{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Generating Text with LSTMs using Word2vec"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "c:\\users\\thushan\\documents\\python_virtualenvs\\tensorflow_venv\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n"
     ]
    }
   ],
   "source": [
    "# These are all the modules we'll be using later. Make sure you can import them\n",
    "# before proceeding further.\n",
    "%matplotlib inline\n",
    "from __future__ import print_function\n",
    "import collections\n",
    "import math\n",
    "import numpy as np\n",
    "import os\n",
    "import random\n",
    "import tensorflow as tf\n",
    "import nltk\n",
    "import zipfile\n",
    "from matplotlib import pylab\n",
    "from six.moves import range\n",
    "from six.moves.urllib.request import urlretrieve\n",
    "import tensorflow as tf\n",
    "import csv\n",
    "\n",
    "# I have separated word vector learning algorithm to\n",
    "# separate file as we have already gone through the details\n",
    "# We will be only focusing on the language generation part\n",
    "import word2vec"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Downloading Stories\n",
    "Stories are automatically downloaded from https://www.cs.cmu.edu/~spok/grimmtmp/, if not detected in the disk. The total size of stories is around ~500KB. The dataset consists of 100 stories."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading file:  stories\\001.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\002.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\003.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\004.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\005.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\006.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\007.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\008.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\009.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\010.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\011.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\012.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\013.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\014.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\015.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\016.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\017.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\018.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\019.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\020.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\021.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\022.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\023.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\024.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\025.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\026.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\027.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\028.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\029.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\030.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\031.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\032.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\033.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\034.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\035.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\036.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\037.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\038.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\039.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\040.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\041.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\042.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\043.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\044.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\045.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\046.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\047.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\048.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\049.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\050.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\051.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\052.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\053.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\054.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\055.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\056.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\057.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\058.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\059.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\060.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\061.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\062.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\063.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\064.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\065.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\066.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\067.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\068.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\069.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\070.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\071.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\072.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\073.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\074.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\075.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\076.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\077.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\078.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\079.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\080.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\081.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\082.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\083.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\084.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\085.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\086.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\087.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\088.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\089.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\090.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\091.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\092.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\093.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\094.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\095.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\096.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\097.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\098.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\099.txt\n",
      "Not downloading. File already exists.\n",
      "Downloading file:  stories\\100.txt\n",
      "Not downloading. File already exists.\n"
     ]
    }
   ],
   "source": [
    "url = 'https://www.cs.cmu.edu/~spok/grimmtmp/'\n",
    "\n",
    "# Create a directory if needed\n",
    "dir_name = 'stories'\n",
    "if not os.path.exists(dir_name):\n",
    "    os.mkdir(dir_name)\n",
    "    \n",
    "def maybe_download(filename):\n",
    "  \"\"\"Download a file if not present\"\"\"\n",
    "  print('Downloading file: ', dir_name+ os.sep+filename)\n",
    "    \n",
    "  if not os.path.exists(dir_name+os.sep+filename):\n",
    "    filename, _ = urlretrieve(url + filename, dir_name+os.sep+filename)\n",
    "  else:\n",
    "    print('Not downloading. File already exists.')\n",
    "  statinfo = os.stat(dir_name+os.sep+filename)\n",
    "  \n",
    "  return filename\n",
    "\n",
    "num_files = 100\n",
    "filenames = [format(i, '03d')+'.txt' for i in range(1,num_files+1)]\n",
    "\n",
    "for fn in filenames:\n",
    "    maybe_download(fn)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100 files found.\n"
     ]
    }
   ],
   "source": [
    "for i in range(len(filenames)):\n",
    "    file_exists = os.path.isfile(os.path.join(dir_name,filenames[i]))\n",
    "    assert file_exists\n",
    "print('%d files found.'%len(filenames))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Reading data\n",
    "Data will be stored in a list of lists where the each list represents a document and document is a list of words. We will then break the text into bigrams"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Processing file stories\\001.txt\n",
      "Data size (Characters) (Document 0) 1693\n",
      "Sample string (Document 0) ['in', 'olden', 'times', 'when', 'wishing', 'still', 'helped', 'one', ',', 'there', 'lived', 'a', 'king', 'whose', 'daughters', 'were', 'all', 'beautiful', ',', 'but', 'the', 'youngest', 'was', 'so', 'beautiful', 'that', 'the', 'sun', 'itself', ',', 'which', 'has', 'seen', 'so', 'much', ',', 'was', 'astonished', 'whenever', 'it', 'shone', 'in', 'her', 'face', '.', 'close', 'by', 'the', 'king', \"'s\"]\n",
      "\n",
      "Processing file stories\\002.txt\n",
      "Data size (Characters) (Document 1) 2167\n",
      "Sample string (Document 1) ['hard', 'by', 'a', 'great', 'forest', 'dwelt', 'a', 'wood-cutter', 'with', 'his', 'wife', ',', 'who', 'had', 'an', 'only', 'child', ',', 'a', 'little', 'girl', 'three', 'years', 'old', '.', 'they', 'were', 'so', 'poor', ',', 'however', ',', 'that', 'they', 'no', 'longer', 'had', 'daily', 'bread', ',', 'and', 'did', 'not', 'know', 'how', 'to', 'get', 'food', 'for', 'her']\n",
      "\n",
      "Processing file stories\\003.txt\n",
      "Data size (Characters) (Document 2) 4499\n",
      "Sample string (Document 2) ['a', 'certain', 'father', 'had', 'two', 'sons', ',', 'the', 'elder', 'of', 'whom', 'was', 'smart', 'and', 'sensible', ',', 'and', 'could', 'do', 'everything', ',', 'but', 'the', 'younger', 'was', 'stupid', 'and', 'could', 'neither', 'learn', 'nor', 'understand', 'anything', ',', 'and', 'when', 'people', 'saw', 'him', 'they', 'said', \"'there\", \"'s\", 'a', 'fellow', 'who', 'will', 'give', 'his', 'father']\n",
      "\n",
      "Processing file stories\\004.txt\n",
      "Data size (Characters) (Document 3) 1265\n",
      "Sample string (Document 3) ['there', 'was', 'once', 'upon', 'a', 'time', 'an', 'old', 'goat', 'who', 'had', 'seven', 'little', 'kids', ',', 'and', 'loved', 'them', 'with', 'all', 'the', 'love', 'of', 'a', 'mother', 'for', 'her', 'children', '.', 'one', 'day', 'she', 'wanted', 'to', 'go', 'into', 'the', 'forest', 'and', 'fetch', 'some', 'food', '.', 'so', 'she', 'called', 'all', 'seven', 'to', 'her']\n",
      "\n",
      "Processing file stories\\005.txt\n",
      "Data size (Characters) (Document 4) 3590\n",
      "Sample string (Document 4) ['there', 'was', 'once', 'upon', 'a', 'time', 'an', 'old', 'king', 'who', 'was', 'ill', 'and', 'thought', 'to', 'himself', \"'i\", 'am', 'lying', 'on', 'what', 'must', 'be', 'my', 'deathbed', '.', \"'\", 'then', 'said', 'he', \"'tell\", 'faithful', 'john', 'to', 'come', 'to', 'me', '.', \"'\", 'faithful', 'john', 'was', 'his', 'favorite', 'servant', ',', 'and', 'was', 'so', 'called']\n",
      "\n",
      "Processing file stories\\006.txt\n",
      "Data size (Characters) (Document 5) 2005\n",
      "Sample string (Document 5) ['there', 'was', 'once', 'a', 'peasant', 'who', 'had', 'driven', 'his', 'cow', 'to', 'the', 'fair', ',', 'and', 'sold', 'her', 'for', 'seven', 'talers', '.', 'on', 'the', 'way', 'home', 'he', 'had', 'to', 'pass', 'a', 'pond', ',', 'and', 'already', 'from', 'afar', 'he', 'heard', 'the', 'frogs', 'crying', ',', 'aik', ',', 'aik', ',', 'aik', ',', 'aik', '.']\n",
      "\n",
      "Processing file stories\\007.txt\n",
      "Data size (Characters) (Document 6) 2265\n",
      "Sample string (Document 6) ['there', 'were', 'once', 'upon', 'a', 'time', 'a', 'king', 'and', 'a', 'queen', 'who', 'lived', 'happily', 'together', 'and', 'had', 'twelve', 'children', ',', 'but', 'they', 'were', 'all', 'boys', '.', 'then', 'said', 'the', 'king', 'to', 'his', 'wife', ',', 'if', 'the', 'thirteenth', 'child', 'which', 'you', 'are', 'about', 'to', 'bring', 'into', 'the', 'world', ',', 'is', 'a']\n",
      "\n",
      "Processing file stories\\008.txt\n",
      "Data size (Characters) (Document 7) 2781\n",
      "Sample string (Document 7) ['little', 'brother', 'took', 'his', 'little', 'sister', 'by', 'the', 'hand', 'and', 'said', ',', 'since', 'our', 'mother', 'died', 'we', 'have', 'had', 'no', 'happiness', '.', 'our', 'step-mother', 'beats', 'us', 'every', 'day', ',', 'and', 'if', 'we', 'come', 'near', 'her', 'she', 'kicks', 'us', 'away', 'with', 'her', 'foot', '.', 'our', 'meals', 'are', 'the', 'hard', 'crusts', 'of']\n",
      "\n",
      "Processing file stories\\009.txt\n",
      "Data size (Characters) (Document 8) 1635\n",
      "Sample string (Document 8) ['there', 'were', 'once', 'a', 'man', 'and', 'a', 'woman', 'who', 'had', 'long', 'in', 'vain', 'wished', 'for', 'a', 'child', '.', 'at', 'length', 'the', 'woman', 'hoped', 'that', 'god', 'was', 'about', 'to', 'grant', 'her', 'desire', '.', 'these', 'people', 'had', 'a', 'little', 'window', 'at', 'the', 'back', 'of', 'their', 'house', 'from', 'which', 'a', 'splendid', 'garden', 'could']\n",
      "\n",
      "Processing file stories\\010.txt\n",
      "Data size (Characters) (Document 9) 2353\n",
      "Sample string (Document 9) ['there', 'was', 'once', 'a', 'man', 'whose', 'wife', 'died', ',', 'and', 'a', 'woman', 'whose', 'husband', 'died', ',', 'and', 'the', 'man', 'had', 'a', 'daughter', ',', 'and', 'the', 'woman', 'also', 'had', 'a', 'daughter', '.', 'the', 'girls', 'were', 'acquainted', 'with', 'each', 'other', ',', 'and', 'went', 'out', 'walking', 'together', ',', 'and', 'afterwards', 'came', 'to', 'the']\n",
      "\n",
      "Processing file stories\\011.txt\n",
      "Data size (Characters) (Document 10) 1038\n",
      "Sample string (Document 10) ['there', 'was', 'once', 'a', 'girl', 'who', 'was', 'idle', 'and', 'would', 'not', 'spin', ',', 'and', 'let', 'her', 'mother', 'say', 'what', 'she', 'would', ',', 'she', 'could', 'not', 'bring', 'her', 'to', 'it', '.', 'at', 'last', 'the', 'mother', 'was', 'once', 'so', 'overcome', 'with', 'anger', 'and', 'impatience', ',', 'that', 'she', 'beat', 'her', ',', 'at', 'which']\n",
      "\n",
      "Processing file stories\\012.txt\n",
      "Data size (Characters) (Document 11) 3427\n",
      "Sample string (Document 11) ['hard', 'by', 'a', 'great', 'forest', 'dwelt', 'a', 'poor', 'wood-cutter', 'with', 'his', 'wife', 'and', 'his', 'two', 'children', '.', 'the', 'boy', 'was', 'called', 'hansel', 'and', 'the', 'girl', 'gretel', '.', 'he', 'had', 'little', 'to', 'bite', 'and', 'to', 'break', ',', 'and', 'once', 'when', 'great', 'dearth', 'fell', 'on', 'the', 'land', ',', 'he', 'could', 'no', 'longer']\n",
      "\n",
      "Processing file stories\\013.txt\n",
      "Data size (Characters) (Document 12) 1619\n",
      "Sample string (Document 12) ['there', 'was', 'once', 'on', 'a', 'time', 'a', 'poor', 'man', ',', 'who', 'could', 'no', 'longer', 'support', 'his', 'only', 'son', '.', 'then', 'said', 'the', 'son', ',', 'dear', 'father', ',', 'things', 'go', 'so', 'badly', 'with', 'us', 'that', 'i', 'am', 'a', 'burden', 'to', 'you', '.', 'i', 'would', 'rather', 'go', 'away', 'and', 'see', 'how', 'i']\n",
      "\n",
      "Processing file stories\\014.txt\n",
      "Data size (Characters) (Document 13) 1813\n",
      "Sample string (Document 13) ['a', 'long', 'time', 'ago', 'there', 'lived', 'a', 'king', 'who', 'was', 'famed', 'for', 'his', 'wisdom', 'through', 'all', 'the', 'land', '.', 'nothing', 'was', 'hidden', 'from', 'him', ',', 'and', 'it', 'seemed', 'as', 'if', 'news', 'of', 'the', 'most', 'secret', 'things', 'was', 'brought', 'to', 'him', 'through', 'the', 'air', '.', 'but', 'he', 'had', 'a', 'strange', 'custom']\n",
      "\n",
      "Processing file stories\\015.txt\n",
      "Data size (Characters) (Document 14) 3834\n",
      "Sample string (Document 14) ['one', 'summer', \"'s\", 'morning', 'a', 'little', 'tailor', 'was', 'sitting', 'on', 'his', 'table', 'by', 'the', 'window', ',', 'he', 'was', 'in', 'good', 'spirits', ',', 'and', 'sewed', 'with', 'all', 'his', 'might', '.', 'then', 'came', 'a', 'peasant', 'woman', 'down', 'the', 'street', 'crying', ',', 'good', 'jams', ',', 'cheap', '.', 'good', 'jams', ',', 'cheap', '.', 'this']\n",
      "\n",
      "Processing file stories\\016.txt\n",
      "Data size (Characters) (Document 15) 3037\n",
      "Sample string (Document 15) ['cinderella', 'the', 'wife', 'of', 'a', 'rich', 'man', 'fell', 'sick', ',', 'and', 'as', 'she', 'felt', 'that', 'her', 'end', 'was', 'drawing', 'near', ',', 'she', 'called', 'her', 'only', 'daughter', 'to', 'her', 'bedside', 'and', 'said', ',', 'dear', 'child', ',', 'be', 'good', 'and', 'pious', ',', 'and', 'then', 'the', 'good', 'god', 'will', 'always', 'protect', 'you', ',']\n",
      "\n",
      "Processing file stories\\017.txt\n",
      "Data size (Characters) (Document 16) 1320\n",
      "Sample string (Document 16) ['there', 'was', 'once', 'a', 'king', \"'s\", 'son', 'who', 'was', 'seized', 'with', 'a', 'desire', 'to', 'travel', 'about', 'the', 'world', ',', 'and', 'took', 'no', 'one', 'with', 'him', 'but', 'a', 'faithful', 'servant', '.', 'one', 'day', 'he', 'came', 'to', 'a', 'great', 'forest', ',', 'and', 'when', 'darkness', 'overtook', 'him', 'he', 'could', 'find', 'no', 'shelter', ',']\n",
      "\n",
      "Processing file stories\\018.txt\n",
      "Data size (Characters) (Document 17) 1370\n",
      "Sample string (Document 17) ['there', 'was', 'once', 'a', 'widow', 'who', 'had', 'two', 'daughters', '-', 'one', 'of', 'whom', 'was', 'pretty', 'and', 'industrious', ',', 'whilst', 'the', 'other', 'was', 'ugly', 'and', 'idle', '.', 'but', 'she', 'was', 'much', 'fonder', 'of', 'the', 'ugly', 'and', 'idle', 'one', ',', 'because', 'she', 'was', 'her', 'own', 'daughter', '.', 'and', 'the', 'other', ',', 'who']\n",
      "\n",
      "Processing file stories\\019.txt\n",
      "Data size (Characters) (Document 18) 1071\n",
      "Sample string (Document 18) ['there', 'was', 'once', 'a', 'man', 'who', 'had', 'seven', 'sons', ',', 'and', 'still', 'he', 'had', 'no', 'daughter', ',', 'however', 'much', 'he', 'wished', 'for', 'one', '.', 'at', 'length', 'his', 'wife', 'again', 'gave', 'him', 'hope', 'of', 'a', 'child', ',', 'and', 'when', 'it', 'came', 'into', 'the', 'world', 'it', 'was', 'a', 'girl', '.', 'the', 'joy']\n",
      "\n",
      "Processing file stories\\020.txt\n",
      "Data size (Characters) (Document 19) 1662\n",
      "Sample string (Document 19) ['little', 'red-cap', 'once', 'upon', 'a', 'time', 'there', 'was', 'a', 'dear', 'little', 'girl', 'who', 'was', 'loved', 'by', 'every', 'one', 'who', 'looked', 'at', 'her', ',', 'but', 'most', 'of', 'all', 'by', 'her', 'grandmother', ',', 'and', 'there', 'was', 'nothing', 'that', 'she', 'would', 'not', 'have', 'given', 'to', 'the', 'child', '.', 'once', 'she', 'gave', 'her', 'a']\n",
      "\n",
      "Processing file stories\\021.txt\n",
      "Data size (Characters) (Document 20) 835\n",
      "Sample string (Document 20) ['in', 'a', 'certain', 'country', 'there', 'was', 'once', 'great', 'lamentation', 'over', 'a', 'wild', 'boar', 'that', 'laid', 'waste', 'the', 'farmer', \"'s\", 'fields', ',', 'killed', 'the', 'cattle', ',', 'and', 'ripped', 'up', 'people', \"'s\", 'bodies', 'with', 'his', 'tusks', '.', 'the', 'king', 'promised', 'a', 'large', 'reward', 'to', 'anyone', 'who', 'would', 'free', 'the', 'land', 'from', 'this']\n",
      "\n",
      "Processing file stories\\022.txt\n",
      "Data size (Characters) (Document 21) 2969\n",
      "Sample string (Document 21) ['there', 'was', 'once', 'a', 'poor', 'woman', 'who', 'gave', 'birth', 'to', 'a', 'little', 'son', ',', 'and', 'as', 'he', 'came', 'into', 'the', 'world', 'with', 'a', 'caul', 'on', ',', 'it', 'was', 'predicted', 'that', 'in', 'his', 'fourteenth', 'year', 'he', 'would', 'have', 'the', 'king', \"'s\", 'daughter', 'for', 'his', 'wife', '.', 'it', 'happened', 'that', 'soon', 'afterwards']\n",
      "\n",
      "Processing file stories\\023.txt\n",
      "Data size (Characters) (Document 22) 2717\n",
      "Sample string (Document 22) ['a', 'certain', 'miller', 'had', 'little', 'by', 'little', 'fallen', 'into', 'poverty', ',', 'and', 'had', 'nothing', 'left', 'but', 'his', 'mill', 'and', 'a', 'large', 'apple-tree', 'behind', 'it', '.', 'once', 'when', 'he', 'had', 'gone', 'into', 'the', 'forest', 'to', 'fetch', 'wood', ',', 'an', 'old', 'man', 'stepped', 'up', 'to', 'him', 'whom', 'he', 'had', 'never', 'seen', 'before']\n",
      "\n",
      "Processing file stories\\024.txt\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data size (Characters) (Document 23) 1202\n",
      "Sample string (Document 23) ['the', 'mother', 'of', 'hans', 'said', ',', 'whither', 'away', ',', 'hans', '.', 'hans', 'answered', ',', 'to', 'gretel', '.', 'behave', 'well', ',', 'hans', '.', 'oh', ',', 'i', \"'ll\", 'behave', 'well', '.', 'good-bye', ',', 'mother', '.', 'good-bye', ',', 'hans', '.', 'hans', 'comes', 'to', 'gretel', '.', 'good', 'day', ',', 'gretel', '.', 'good', 'day', ',']\n",
      "\n",
      "Processing file stories\\025.txt\n",
      "Data size (Characters) (Document 24) 1090\n",
      "Sample string (Document 24) ['an', 'aged', 'count', 'once', 'lived', 'in', 'switzerland', ',', 'who', 'had', 'an', 'only', 'son', ',', 'but', 'he', 'was', 'stupid', ',', 'and', 'could', 'learn', 'nothing', '.', 'then', 'said', 'the', 'father', ',', 'hark', 'you', ',', 'my', 'son', ',', 'try', 'as', 'i', 'will', 'i', 'can', 'get', 'nothing', 'into', 'your', 'head', '.', 'you', 'must', 'go']\n",
      "\n",
      "Processing file stories\\026.txt\n",
      "Data size (Characters) (Document 25) 1550\n",
      "Sample string (Document 25) ['there', 'was', 'once', 'a', 'man', 'who', 'had', 'a', 'daughter', 'who', 'was', 'called', 'clever', 'elsie', '.', 'and', 'when', 'she', 'had', 'grown', 'up', 'her', 'father', 'said', ',', 'we', 'will', 'get', 'her', 'married', '.', 'yes', ',', 'said', 'the', 'mother', ',', 'if', 'only', 'someone', 'would', 'come', 'who', 'would', 'have', 'her', '.', 'at', 'length', 'a']\n",
      "\n",
      "Processing file stories\\027.txt\n",
      "Data size (Characters) (Document 26) 4447\n",
      "Sample string (Document 26) ['there', 'was', 'once', 'upon', 'a', 'time', 'a', 'tailor', 'who', 'had', 'three', 'sons', ',', 'and', 'only', 'one', 'goat', '.', 'but', 'as', 'the', 'goat', 'supported', 'all', 'of', 'them', 'with', 'her', 'milk', ',', 'she', 'was', 'obliged', 'to', 'have', 'good', 'food', ',', 'and', 'to', 'be', 'taken', 'every', 'day', 'to', 'pasture', '.', 'the', 'sons', 'did']\n",
      "\n",
      "Processing file stories\\028.txt\n",
      "Data size (Characters) (Document 27) 2638\n",
      "Sample string (Document 27) ['there', 'was', 'once', 'a', 'poor', 'peasant', 'who', 'sat', 'in', 'the', 'evening', 'by', 'the', 'hearth', 'and', 'poked', 'the', 'fire', ',', 'and', 'his', 'wife', 'sat', 'and', 'spun', '.', 'then', 'said', 'he', ',', 'how', 'sad', 'it', 'is', 'that', 'we', 'have', 'no', 'children', '.', 'with', 'us', 'all', 'is', 'so', 'quiet', ',', 'and', 'in', 'other']\n",
      "\n",
      "Processing file stories\\029.txt\n",
      "Data size (Characters) (Document 28) 575\n",
      "Sample string (Document 28) ['there', 'was', 'once', 'a', 'poor', 'servant-girl', 'who', 'was', 'industrious', 'and', 'cleanly', 'and', 'swept', 'the', 'house', 'every', 'day', ',', 'and', 'emptied', 'her', 'sweepings', 'on', 'the', 'great', 'heap', 'in', 'front', 'of', 'the', 'door', '.', 'one', 'morning', 'when', 'she', 'was', 'just', 'going', 'back', 'to', 'her', 'work', ',', 'she', 'found', 'a', 'letter', 'on', 'this']\n",
      "\n",
      "Processing file stories\\030.txt\n",
      "Data size (Characters) (Document 29) 1611\n",
      "Sample string (Document 29) ['there', 'was', 'once', 'upon', 'a', 'time', 'a', 'miller', ',', 'who', 'had', 'a', 'beautiful', 'daughter', ',', 'and', 'as', 'she', 'was', 'grown', 'up', ',', 'he', 'wished', 'that', 'she', 'was', 'provided', 'for', ',', 'and', 'well', 'married', '.', 'he', 'thought', ',', 'if', 'any', 'good', 'suitor', 'comes', 'and', 'asks', 'for', 'her', ',', 'i', 'will', 'give']\n",
      "\n",
      "Processing file stories\\031.txt\n",
      "Data size (Characters) (Document 30) 722\n",
      "Sample string (Document 30) ['a', 'poor', 'man', 'had', 'so', 'many', 'children', 'that', 'he', 'had', 'already', 'asked', 'everyone', 'in', 'the', 'world', 'to', 'be', 'godfather', ',', 'and', 'when', 'still', 'another', 'child', 'was', 'born', ',', 'no', 'one', 'else', 'was', 'left', 'whom', 'he', 'could', 'invite', '.', 'he', 'knew', 'not', 'what', 'to', 'do', ',', 'and', ',', 'in', 'his', 'perplexity']\n",
      "\n",
      "Processing file stories\\032.txt\n",
      "Data size (Characters) (Document 31) 354\n",
      "Sample string (Document 31) ['there', 'was', 'once', 'a', 'little', 'girl', 'who', 'was', 'obstinate', 'and', 'inquisitive', ',', 'and', 'when', 'her', 'parents', 'told', 'her', 'to', 'do', 'anything', ',', 'she', 'did', 'not', 'obey', 'them', ',', 'so', 'how', 'could', 'she', 'fare', 'well', '.', 'one', 'day', 'she', 'said', 'to', 'her', 'parents', ',', 'i', 'have', 'heard', 'so', 'much', 'of', 'frau']\n",
      "\n",
      "Processing file stories\\033.txt\n",
      "Data size (Characters) (Document 32) 1404\n",
      "Sample string (Document 32) ['a', 'poor', 'man', 'had', 'twelve', 'children', 'and', 'was', 'forced', 'to', 'work', 'night', 'and', 'day', 'to', 'give', 'them', 'even', 'bread', '.', 'when', 'therefore', 'the', 'thirteenth', 'came', 'into', 'the', 'world', ',', 'he', 'knew', 'not', 'what', 'to', 'do', 'in', 'his', 'trouble', ',', 'but', 'ran', 'out', 'into', 'the', 'great', 'highway', ',', 'and', 'resolved', 'to']\n",
      "\n",
      "Processing file stories\\034.txt\n",
      "Data size (Characters) (Document 33) 1835\n",
      "Sample string (Document 33) ['a', 'certain', 'tailor', 'had', 'a', 'son', ',', 'who', 'happened', 'to', 'be', 'small', ',', 'and', 'no', 'bigger', 'than', 'a', 'thumb', ',', 'and', 'on', 'this', 'account', 'he', 'was', 'always', 'called', 'thumbling', '.', 'he', 'had', ',', 'however', ',', 'some', 'courage', 'in', 'him', ',', 'and', 'said', 'to', 'his', 'father', ',', 'father', ',', 'i', 'must']\n",
      "\n",
      "Processing file stories\\035.txt\n",
      "Data size (Characters) (Document 34) 1614\n",
      "Sample string (Document 34) ['there', 'was', 'once', 'a', 'wizard', 'who', 'used', 'to', 'take', 'the', 'form', 'of', 'a', 'poor', 'man', ',', 'and', 'went', 'to', 'houses', 'and', 'begged', ',', 'and', 'caught', 'pretty', 'girls', '.', 'no', 'one', 'knew', 'whither', 'he', 'carried', 'them', ',', 'for', 'they', 'were', 'never', 'seen', 'again', '.', 'one', 'day', 'he', 'appeared', 'before', 'the', 'door']\n",
      "\n",
      "Processing file stories\\036.txt\n",
      "Data size (Characters) (Document 35) 3766\n",
      "Sample string (Document 35) ['it', 'is', 'now', 'long', 'ago', ',', 'quite', 'two', 'thousand', 'years', ',', 'since', 'there', 'was', 'a', 'rich', 'man', 'who', 'had', 'a', 'beautiful', 'and', 'pious', 'wife', ',', 'and', 'they', 'loved', 'each', 'other', 'dearly', '.', 'they', 'had', ',', 'however', ',', 'no', 'children', ',', 'though', 'they', 'wished', 'for', 'them', 'very', 'much', ',', 'and', 'the']\n",
      "\n",
      "Processing file stories\\037.txt\n",
      "Data size (Characters) (Document 36) 984\n",
      "Sample string (Document 36) ['a', 'farmer', 'once', 'had', 'a', 'faithful', 'dog', 'called', 'sultan', ',', 'who', 'had', 'grown', 'old', ',', 'and', 'lost', 'all', 'his', 'teeth', ',', 'so', 'that', 'he', 'could', 'no', 'longer', 'hold', 'on', 'to', 'anything', '.', 'one', 'day', 'the', 'farmer', 'was', 'standing', 'with', 'his', 'wife', 'before', 'the', 'house-door', ',', 'and', 'said', ',', 'to-morrow', 'i']\n",
      "\n",
      "Processing file stories\\038.txt\n",
      "Data size (Characters) (Document 37) 2250\n",
      "Sample string (Document 37) ['once', 'upon', 'a', 'time', ',', 'a', 'certain', 'king', 'was', 'hunting', 'in', 'a', 'great', 'forest', ',', 'and', 'he', 'chased', 'a', 'wild', 'beast', 'so', 'eagerly', 'that', 'none', 'of', 'his', 'attendants', 'could', 'follow', 'him', '.', 'when', 'evening', 'drew', 'near', 'he', 'stopped', 'and', 'looked', 'around', 'him', ',', 'and', 'then', 'he', 'saw', 'that', 'he', 'had']\n",
      "\n",
      "Processing file stories\\039.txt\n",
      "Data size (Characters) (Document 38) 1505\n",
      "Sample string (Document 38) ['briar-rose', 'a', 'long', 'time', 'ago', 'there', 'were', 'a', 'king', 'and', 'queen', 'who', 'said', 'every', 'day', ',', 'ah', ',', 'if', 'only', 'we', 'had', 'a', 'child', ',', 'but', 'they', 'never', 'had', 'one', '.', 'but', 'it', 'happened', 'that', 'once', 'when', 'the', 'queen', 'was', 'bathing', ',', 'a', 'frog', 'crept', 'out', 'of', 'the', 'water', 'on']\n",
      "\n",
      "Processing file stories\\040.txt\n",
      "Data size (Characters) (Document 39) 1115\n",
      "Sample string (Document 39) ['there', 'was', 'once', 'a', 'forester', 'who', 'went', 'into', 'the', 'forest', 'to', 'hunt', ',', 'and', 'as', 'he', 'entered', 'it', 'he', 'heard', 'a', 'sound', 'of', 'screaming', 'as', 'if', 'a', 'little', 'child', 'were', 'there', '.', 'he', 'followed', 'the', 'sound', ',', 'and', 'at', 'last', 'came', 'to', 'a', 'high', 'tree', ',', 'and', 'at', 'the', 'top']\n",
      "\n",
      "Processing file stories\\041.txt\n",
      "Data size (Characters) (Document 40) 1886\n",
      "Sample string (Document 40) ['a', 'king', 'had', 'a', 'daughter', 'who', 'was', 'beautiful', 'beyond', 'all', 'measure', ',', 'but', 'so', 'proud', 'and', 'haughty', 'withal', 'that', 'no', 'suitor', 'was', 'good', 'enough', 'for', 'her', '.', 'she', 'sent', 'away', 'one', 'after', 'the', 'other', ',', 'and', 'ridiculed', 'them', 'as', 'well', '.', 'once', 'the', 'king', 'made', 'a', 'great', 'feast', 'and', 'invited']\n",
      "\n",
      "Processing file stories\\042.txt\n",
      "Data size (Characters) (Document 41) 3616\n",
      "Sample string (Document 41) ['snow', 'white', 'and', 'the', 'seven', 'dwarfs', 'once', 'upon', 'a', 'time', 'in', 'the', 'middle', 'of', 'winter', ',', 'when', 'the', 'flakes', 'of', 'snow', 'were', 'falling', 'like', 'feathers', 'from', 'the', 'sky', ',', 'a', 'queen', 'sat', 'at', 'a', 'window', 'sewing', ',', 'and', 'the', 'frame', 'of', 'the', 'window', 'was', 'made', 'of', 'black', 'ebony', '.', 'and']\n",
      "\n",
      "Processing file stories\\043.txt\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data size (Characters) (Document 42) 2662\n",
      "Sample string (Document 42) ['there', 'were', 'once', 'three', 'brothers', 'who', 'had', 'fallen', 'deeper', 'and', 'deeper', 'into', 'poverty', ',', 'and', 'at', 'last', 'their', 'need', 'was', 'so', 'great', 'that', 'they', 'had', 'to', 'endure', 'hunger', ',', 'and', 'had', 'nothing', 'to', 'eat', 'or', 'drink', '.', 'then', 'said', 'they', ',', 'it', 'can', 'not', 'go', 'on', 'like', 'this', ',', 'we']\n",
      "\n",
      "Processing file stories\\044.txt\n",
      "Data size (Characters) (Document 43) 1266\n",
      "Sample string (Document 43) ['rumpelstiltskin', 'once', 'there', 'was', 'a', 'miller', 'who', 'was', 'poor', ',', 'but', 'who', 'had', 'a', 'beautiful', 'daughter', '.', 'now', 'it', 'happened', 'that', 'he', 'had', 'to', 'go', 'and', 'speak', 'to', 'the', 'king', ',', 'and', 'in', 'order', 'to', 'make', 'himself', 'appear', 'important', 'he', 'said', 'to', 'him', ',', 'i', 'have', 'a', 'daughter', 'who', 'can']\n",
      "\n",
      "Processing file stories\\045.txt\n",
      "Data size (Characters) (Document 44) 1709\n",
      "Sample string (Document 44) ['there', 'was', 'once', 'upon', 'a', 'time', 'a', 'woman', 'who', 'was', 'a', 'real', 'witch', 'and', 'had', 'two', 'daughters', ',', 'one', 'ugly', 'and', 'wicked', ',', 'and', 'this', 'one', 'she', 'loved', 'because', 'she', 'was', 'her', 'own', 'daughter', ',', 'and', 'one', 'beautiful', 'and', 'good', ',', 'and', 'this', 'one', 'she', 'hated', ',', 'because', 'she', 'was']\n",
      "\n",
      "Processing file stories\\046.txt\n",
      "Data size (Characters) (Document 45) 3493\n",
      "Sample string (Document 45) ['in', 'olden', 'times', 'there', 'was', 'a', 'king', ',', 'who', 'had', 'behind', 'his', 'palace', 'a', 'beautiful', 'pleasure-garden', 'in', 'which', 'there', 'was', 'a', 'tree', 'that', 'bore', 'golden', 'apples', '.', 'when', 'the', 'apples', 'were', 'getting', 'ripe', 'they', 'were', 'counted', ',', 'but', 'on', 'the', 'very', 'next', 'morning', 'one', 'was', 'missing', '.', 'this', 'was', 'told']\n",
      "\n",
      "Processing file stories\\047.txt\n",
      "Data size (Characters) (Document 46) 9901\n",
      "Sample string (Document 46) ['there', 'were', 'once', 'upon', 'a', 'time', 'two', 'brothers', ',', 'one', 'rich', 'and', 'the', 'other', 'poor', '.', 'the', 'rich', 'one', 'was', 'a', 'goldsmith', 'and', 'evil-hearted', '.', 'the', 'poor', 'one', 'supported', 'himself', 'by', 'making', 'brooms', ',', 'and', 'was', 'good', 'and', 'honorable', '.', 'he', 'had', 'two', 'children', ',', 'who', 'were', 'twin', 'brothers', 'and']\n",
      "\n",
      "Processing file stories\\048.txt\n",
      "Data size (Characters) (Document 47) 934\n",
      "Sample string (Document 47) ['two', 'kings', \"'\", 'sons', 'once', 'went', 'out', 'in', 'search', 'of', 'adventures', ',', 'and', 'fell', 'into', 'a', 'wild', ',', 'disorderly', 'way', 'of', 'living', ',', 'so', 'that', 'they', 'never', 'came', 'home', 'again', '.', 'the', 'youngest', ',', 'who', 'was', 'called', 'simpleton', ',', 'set', 'out', 'to', 'seek', 'his', 'brothers', ',', 'but', 'when', 'at', 'length']\n",
      "\n",
      "Processing file stories\\049.txt\n",
      "Data size (Characters) (Document 48) 1180\n",
      "Sample string (Document 48) ['there', 'was', 'once', 'upon', 'a', 'time', 'a', 'king', 'who', 'had', 'three', 'sons', ',', 'of', 'whom', 'two', 'were', 'clever', 'and', 'wise', ',', 'but', 'the', 'third', 'did', 'not', 'speak', 'much', ',', 'and', 'was', 'simple', ',', 'and', 'was', 'called', 'the', 'simpleton', '.', 'when', 'the', 'king', 'had', 'become', 'old', 'and', 'weak', ',', 'and', 'was']\n",
      "\n",
      "Processing file stories\\050.txt\n",
      "Data size (Characters) (Document 49) 1792\n",
      "Sample string (Document 49) ['there', 'was', 'a', 'man', 'who', 'had', 'three', 'sons', ',', 'the', 'youngest', 'of', 'whom', 'was', 'called', 'dummling', ',', 'and', 'was', 'despised', ',', 'mocked', ',', 'and', 'sneered', 'at', 'on', 'every', 'occasion', '.', 'it', 'happened', 'that', 'the', 'eldest', 'wanted', 'to', 'go', 'into', 'the', 'forest', 'to', 'hew', 'wood', ',', 'and', 'before', 'he', 'went', 'his']\n",
      "\n",
      "Processing file stories\\051.txt\n",
      "Data size (Characters) (Document 50) 2503\n",
      "Sample string (Document 50) ['allerleirauh', 'there', 'was', 'once', 'upon', 'a', 'time', 'a', 'king', 'who', 'had', 'a', 'wife', 'with', 'golden', 'hair', ',', 'and', 'she', 'was', 'so', 'beautiful', 'that', 'her', 'equal', 'was', 'not', 'to', 'be', 'found', 'on', 'earth', '.', 'it', 'came', 'to', 'pass', 'that', 'she', 'lay', 'ill', ',', 'and', 'as', 'she', 'felt', 'that', 'she', 'must', 'soon']\n",
      "\n",
      "Processing file stories\\052.txt\n",
      "Data size (Characters) (Document 51) 579\n",
      "Sample string (Document 51) ['there', 'was', 'once', 'a', 'woman', 'and', 'her', 'daughter', 'who', 'lived', 'in', 'a', 'pretty', 'garden', 'with', 'cabbages', '.', 'and', 'a', 'little', 'hare', 'came', 'into', 'it', ',', 'and', 'during', 'the', 'winter', 'time', 'ate', 'all', 'the', 'cabbages', '.', 'then', 'says', 'the', 'mother', 'to', 'the', 'daughter', ',', 'go', 'into', 'the', 'garden', ',', 'and', 'chase']\n",
      "\n",
      "Processing file stories\\053.txt\n",
      "Data size (Characters) (Document 52) 1249\n",
      "Sample string (Document 52) ['there', 'was', 'once', 'a', 'king', \"'s\", 'son', 'who', 'had', 'a', 'bride', 'whom', 'he', 'loved', 'very', 'much', '.', 'and', 'when', 'he', 'was', 'sitting', 'beside', 'her', 'and', 'very', 'happy', ',', 'news', 'came', 'that', 'his', 'father', 'lay', 'sick', 'unto', 'death', ',', 'and', 'desired', 'to', 'see', 'him', 'once', 'again', 'before', 'his', 'end', '.', 'then']\n",
      "\n",
      "Processing file stories\\054.txt\n",
      "Data size (Characters) (Document 53) 890\n",
      "Sample string (Document 53) ['hans', 'wished', 'to', 'put', 'his', 'son', 'to', 'learn', 'a', 'trade', ',', 'so', 'he', 'went', 'into', 'the', 'church', 'and', 'prayed', 'to', 'our', 'lord', 'god', 'to', 'know', 'which', 'would', 'be', 'the', 'most', 'suitable', 'for', 'him', '.', 'then', 'the', 'clerk', 'got', 'behind', 'the', 'altar', ',', 'and', 'said', ',', 'thieving', ',', 'thieving', '.', 'on']\n",
      "\n",
      "Processing file stories\\055.txt\n",
      "Data size (Characters) (Document 54) 1117\n",
      "Sample string (Document 54) ['a', 'father', 'once', 'called', 'his', 'three', 'sons', 'before', 'him', ',', 'and', 'he', 'gave', 'to', 'the', 'first', 'a', 'cock', ',', 'to', 'the', 'second', 'a', 'scythe', ',', 'and', 'to', 'the', 'third', 'a', 'cat', '.', 'i', 'am', 'already', 'aged', ',', 'said', 'he', ',', 'my', 'death', 'is', 'nigh', ',', 'and', 'i', 'have', 'wished', 'to']\n",
      "\n",
      "Processing file stories\\056.txt\n",
      "Data size (Characters) (Document 55) 2386\n",
      "Sample string (Document 55) ['there', 'was', 'once', 'a', 'man', 'who', 'understood', 'all', 'kinds', 'of', 'arts', '.', 'he', 'served', 'in', 'war', ',', 'and', 'behaved', 'well', 'and', 'bravely', ',', 'but', 'when', 'the', 'war', 'was', 'over', 'he', 'received', 'his', 'dismissal', ',', 'and', 'three', 'farthings', 'for', 'his', 'expenses', 'on', 'the', 'way', '.', 'wait', ',', 'said', 'he', ',', 'i']\n",
      "\n",
      "Processing file stories\\057.txt\n",
      "Data size (Characters) (Document 56) 441\n",
      "Sample string (Document 56) ['the', 'she-wolf', 'brought', 'into', 'the', 'world', 'a', 'young', 'one', ',', 'and', 'invited', 'the', 'fox', 'to', 'be', 'godfather', '.', 'after', 'all', ',', 'he', 'is', 'a', 'near', 'relative', 'of', 'ours', ',', 'said', 'she', ',', 'he', 'has', 'a', 'good', 'understanding', ',', 'and', 'much', 'talent', ',', 'he', 'can', 'instruct', 'my', 'little', 'son', ',', 'and']\n",
      "\n",
      "Processing file stories\\058.txt\n",
      "Data size (Characters) (Document 57) 2052\n",
      "Sample string (Document 57) ['there', 'was', 'once', 'upon', 'a', 'time', 'a', 'queen', 'to', 'whom', 'god', 'had', 'given', 'no', 'children', '.', 'every', 'morning', 'she', 'went', 'into', 'the', 'garden', 'and', 'prayed', 'to', 'god', 'in', 'heaven', 'to', 'bestow', 'on', 'her', 'a', 'son', 'or', 'a', 'daughter', '.', 'then', 'an', 'angel', 'from', 'heaven', 'came', 'to', 'her', 'and', 'said', ',']\n",
      "\n",
      "Processing file stories\\059.txt\n",
      "Data size (Characters) (Document 58) 280\n",
      "Sample string (Document 58) ['there', 'was', 'once', 'a', 'very', 'old', 'man', ',', 'whose', 'eyes', 'had', 'become', 'dim', ',', 'his', 'ears', 'dull', 'of', 'hearing', ',', 'his', 'knees', 'trembled', ',', 'and', 'when', 'he', 'sat', 'at', 'table', 'he', 'could', 'hardly', 'hold', 'the', 'spoon', ',', 'and', 'spilt', 'the', 'broth', 'upon', 'the', 'table-cloth', 'or', 'let', 'it', 'run', 'out', 'of']\n",
      "\n",
      "Processing file stories\\060.txt\n",
      "Data size (Characters) (Document 59) 338\n",
      "Sample string (Document 59) ['a', 'little', 'brother', 'and', 'sister', 'were', 'once', 'playing', 'by', 'a', 'well', ',', 'and', 'while', 'they', 'were', 'thus', 'playing', ',', 'they', 'both', 'fell', 'in', '.', 'a', 'water-nixie', 'lived', 'down', 'below', ',', 'who', 'said', ',', 'now', 'i', 'have', 'got', 'you', ',', 'now', 'you', 'shall', 'work', 'hard', 'for', 'me', ',', 'and', 'carried', 'them']\n",
      "\n",
      "Processing file stories\\061.txt\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data size (Characters) (Document 60) 4882\n",
      "Sample string (Document 60) ['there', 'was', 'one', 'upon', 'a', 'time', 'a', 'great', 'war', ',', 'and', 'when', 'it', 'came', 'to', 'an', 'end', ',', 'many', 'soldiers', 'were', 'discharged', '.', 'then', 'brother', 'lustig', 'also', 'received', 'his', 'dismissal', ',', 'and', 'with', 'it', 'nothing', 'but', 'a', 'small', 'loaf', 'of', 'ammunition-bread', ',', 'and', 'four', 'kreuzers', 'in', 'money', ',', 'with', 'which']\n",
      "\n",
      "Processing file stories\\062.txt\n",
      "Data size (Characters) (Document 61) 2355\n",
      "Sample string (Document 61) ['hans', 'had', 'served', 'his', 'master', 'for', 'seven', 'years', ',', 'so', 'he', 'said', 'to', 'him', ',', 'master', ',', 'my', 'time', 'is', 'up', ',', 'now', 'i', 'should', 'be', 'glad', 'to', 'go', 'back', 'home', 'to', 'my', 'mother', ',', 'give', 'me', 'my', 'wages', '.', 'the', 'master', 'answered', ',', 'you', 'have', 'served', 'me', 'faithfully', 'and']\n",
      "\n",
      "Processing file stories\\063.txt\n",
      "Data size (Characters) (Document 62) 512\n",
      "Sample string (Document 62) ['there', 'was', 'once', 'upon', 'a', 'time', 'a', 'young', 'peasant', 'named', 'hans', ',', 'whose', 'uncle', 'wanted', 'to', 'find', 'him', 'a', 'rich', 'wife', '.', 'he', 'therefore', 'seated', 'hans', 'behind', 'the', 'stove', ',', 'and', 'had', 'it', 'made', 'very', 'hot', '.', 'then', 'he', 'fetched', 'a', 'pot', 'of', 'milk', 'and', 'plenty', 'of', 'white', 'bread', ',']\n",
      "\n",
      "Processing file stories\\064.txt\n",
      "Data size (Characters) (Document 63) 2230\n",
      "Sample string (Document 63) ['there', 'was', 'once', 'a', 'poor', 'man', 'and', 'a', 'poor', 'woman', 'who', 'had', 'nothing', 'but', 'a', 'little', 'cottage', ',', 'and', 'who', 'earned', 'their', 'bread', 'by', 'fishing', ',', 'and', 'always', 'lived', 'from', 'hand', 'to', 'mouth', '.', 'but', 'it', 'came', 'to', 'pass', 'one', 'day', 'when', 'the', 'man', 'was', 'sitting', 'by', 'the', 'water-side', ',']\n",
      "\n",
      "Processing file stories\\065.txt\n",
      "Data size (Characters) (Document 64) 2672\n",
      "Sample string (Document 64) ['there', 'was', 'once', 'upon', 'a', 'time', 'a', 'man', 'who', 'was', 'about', 'to', 'set', 'out', 'on', 'a', 'long', 'journey', ',', 'and', 'on', 'parting', 'he', 'asked', 'his', 'three', 'daughters', 'what', 'he', 'should', 'bring', 'back', 'with', 'him', 'for', 'them', '.', 'whereupon', 'the', 'eldest', 'wished', 'for', 'pearls', ',', 'the', 'second', 'wished', 'for', 'diamonds', ',']\n",
      "\n",
      "Processing file stories\\066.txt\n",
      "Data size (Characters) (Document 65) 2543\n",
      "Sample string (Document 65) ['there', 'was', 'once', 'upon', 'a', 'time', 'an', 'old', 'queen', 'whose', 'husband', 'had', 'been', 'dead', 'for', 'many', 'years', ',', 'and', 'she', 'had', 'a', 'beautiful', 'daughter', '.', 'when', 'the', 'princess', 'grew', 'up', 'she', 'was', 'betrothed', 'to', 'a', 'prince', 'who', 'lived', 'at', 'a', 'great', 'distance', '.', 'when', 'the', 'time', 'came', 'for', 'her', 'to']\n",
      "\n",
      "Processing file stories\\067.txt\n",
      "Data size (Characters) (Document 66) 3592\n",
      "Sample string (Document 66) ['once', 'upon', 'a', 'time', 'a', 'countryman', 'had', 'a', 'son', 'who', 'was', 'as', 'big', 'as', 'a', 'thumb', ',', 'and', 'did', 'not', 'become', 'any', 'bigger', ',', 'and', 'during', 'several', 'years', 'did', 'not', 'grow', 'one', 'hair', \"'s\", 'breadth', '.', 'once', 'when', 'the', 'father', 'was', 'going', 'out', 'to', 'plough', ',', 'the', 'little', 'one', 'said']\n",
      "\n",
      "Processing file stories\\068.txt\n",
      "Data size (Characters) (Document 67) 2088\n",
      "Sample string (Document 67) ['there', 'was', 'once', 'upon', 'a', 'time', 'a', 'rich', 'king', 'who', 'had', 'three', 'daughters', ',', 'who', 'daily', 'went', 'to', 'walk', 'in', 'the', 'palace', 'garden', ',', 'and', 'the', 'king', 'was', 'a', 'great', 'lover', 'of', 'all', 'kinds', 'of', 'fine', 'trees', ',', 'but', 'there', 'was', 'one', 'for', 'which', 'he', 'had', 'such', 'an', 'affection', ',']\n",
      "\n",
      "Processing file stories\\069.txt\n",
      "Data size (Characters) (Document 68) 2814\n",
      "Sample string (Document 68) ['there', 'was', 'a', 'certain', 'merchant', 'who', 'had', 'two', 'children', ',', 'a', 'boy', 'and', 'a', 'girl', ',', 'they', 'were', 'both', 'young', ',', 'and', 'could', 'not', 'walk', '.', 'and', 'two', 'richly-laden', 'ships', 'of', 'his', 'sailed', 'forth', 'to', 'sea', 'with', 'all', 'his', 'property', 'on', 'board', ',', 'and', 'just', 'as', 'he', 'was', 'expecting', 'to']\n",
      "\n",
      "Processing file stories\\070.txt\n",
      "Data size (Characters) (Document 69) 2587\n",
      "Sample string (Document 69) ['there', 'was', 'once', 'upon', 'a', 'time', 'a', 'queen', 'who', 'had', 'a', 'little', 'daughter', 'who', 'was', 'still', 'so', 'young', 'that', 'she', 'had', 'to', 'be', 'carried', '.', 'one', 'day', 'the', 'child', 'was', 'naughty', ',', 'and', 'the', 'mother', 'might', 'say', 'what', 'she', 'liked', ',', 'but', 'the', 'child', 'would', 'not', 'be', 'quiet', '.', 'then']\n",
      "\n",
      "Processing file stories\\071.txt\n",
      "Data size (Characters) (Document 70) 1619\n",
      "Sample string (Document 70) ['there', 'was', 'once', 'a', 'poor', 'peasant', 'who', 'had', 'no', 'land', ',', 'but', 'only', 'a', 'small', 'house', ',', 'and', 'one', 'daughter', '.', 'then', 'said', 'the', 'daughter', ',', 'we', 'ought', 'to', 'ask', 'our', 'lord', 'the', 'king', 'for', 'a', 'bit', 'of', 'newly-cleared', 'land', '.', 'when', 'the', 'king', 'heard', 'of', 'their', 'poverty', ',', 'he']\n",
      "\n",
      "Processing file stories\\072.txt\n",
      "Data size (Characters) (Document 71) 1679\n",
      "Sample string (Document 71) ['about', 'a', 'thousand', 'or', 'more', 'years', 'ago', ',', 'there', 'were', 'in', 'this', 'country', 'nothing', 'but', 'small', 'kings', ',', 'and', 'one', 'of', 'them', 'who', 'lived', 'on', 'the', 'keuterberg', 'was', 'very', 'fond', 'of', 'hunting', '.', 'once', 'on', 'a', 'time', 'when', 'he', 'was', 'riding', 'forth', 'from', 'his', 'castle', 'with', 'his', 'huntsmen', ',', 'three']\n",
      "\n",
      "Processing file stories\\073.txt\n",
      "Data size (Characters) (Document 72) 2708\n",
      "Sample string (Document 72) ['there', 'was', 'once', 'a', 'king', 'who', 'had', 'an', 'illness', ',', 'and', 'no', 'one', 'believed', 'that', 'he', 'would', 'come', 'out', 'of', 'it', 'with', 'his', 'life', '.', 'he', 'had', 'three', 'sons', 'who', 'were', 'much', 'distressed', 'about', 'it', ',', 'and', 'went', 'down', 'into', 'the', 'palace-garden', 'and', 'wept', '.', 'there', 'they', 'met', 'an', 'old']\n",
      "\n",
      "Processing file stories\\074.txt\n",
      "Data size (Characters) (Document 73) 2203\n",
      "Sample string (Document 73) ['there', 'was', 'once', 'a', 'poor', 'woodcutter', 'who', 'toiled', 'from', 'early', 'morning', 'till', 'late', 'at', 'night', '.', 'when', 'at', 'last', 'he', 'had', 'laid', 'by', 'some', 'money', 'he', 'said', 'to', 'his', 'boy', ',', '``', 'you', 'are', 'my', 'only', 'child', ',', 'i', 'will', 'spend', 'the', 'money', 'which', 'i', 'have', 'earned', 'with', 'the', 'sweat']\n",
      "\n",
      "Processing file stories\\075.txt\n",
      "Data size (Characters) (Document 74) 1522\n",
      "Sample string (Document 74) ['a', 'discharged', 'soldier', 'had', 'nothing', 'to', 'live', 'on', ',', 'and', 'did', 'not', 'know', 'how', 'to', 'make', 'his', 'way', '.', 'so', 'he', 'went', 'out', 'into', 'the', 'forest', 'and', 'when', 'he', 'had', 'walked', 'for', 'a', 'short', 'time', ',', 'he', 'met', 'a', 'little', 'man', 'who', 'turned', 'out', 'to', 'be', 'the', 'devil', '.', 'the']\n",
      "\n",
      "Processing file stories\\076.txt\n",
      "Data size (Characters) (Document 75) 2345\n",
      "Sample string (Document 75) ['there', 'was', 'once', 'a', 'young', 'fellow', 'who', 'enlisted', 'as', 'a', 'soldier', ',', 'conducted', 'himself', 'bravely', ',', 'and', 'was', 'always', 'the', 'foremost', 'when', 'it', 'rained', 'bullets', '.', 'so', 'long', 'as', 'the', 'war', 'lasted', ',', 'all', 'went', 'well', ',', 'but', 'when', 'peace', 'was', 'made', ',', 'he', 'received', 'his', 'dismissal', ',', 'and', 'the']\n",
      "\n",
      "Processing file stories\\077.txt\n",
      "Data size (Characters) (Document 76) 1108\n",
      "Sample string (Document 76) ['once', 'in', 'summer-time', 'the', 'bear', 'and', 'the', 'wolf', 'were', 'walking', 'in', 'the', 'forest', ',', 'and', 'the', 'bear', 'heard', 'a', 'bird', 'singing', 'so', 'beautifully', 'that', 'he', 'said', ',', \"''\", 'brother', 'wolf', ',', 'what', 'bird', 'is', 'it', 'that', 'sings', 'so', 'well', '?', \"''\", '``', 'that', 'is', 'the', 'king', 'of', 'birds', ',', \"''\"]\n",
      "\n",
      "Processing file stories\\078.txt\n",
      "Data size (Characters) (Document 77) 284\n",
      "Sample string (Document 77) ['there', 'was', 'a', 'poor', 'but', 'good', 'little', 'girl', 'who', 'lived', 'alone', 'with', 'her', 'mother', ',', 'and', 'they', 'no', 'longer', 'had', 'anything', 'to', 'eat', '.', 'so', 'the', 'child', 'went', 'into', 'the', 'forest', ',', 'and', 'there', 'an', 'aged', 'woman', 'met', 'her', 'who', 'was', 'aware', 'of', 'her', 'sorrow', ',', 'and', 'presented', 'her', 'with']\n",
      "\n",
      "Processing file stories\\079.txt\n",
      "Data size (Characters) (Document 78) 1822\n",
      "Sample string (Document 78) ['one', 'day', 'a', 'peasant', 'took', 'his', 'good', 'hazel-stick', 'out', 'of', 'the', 'corner', 'and', 'said', 'to', 'his', 'wife', ',', 'trina', ',', 'i', 'am', 'going', 'across', 'country', ',', 'and', 'shall', 'not', 'return', 'for', 'three', 'days', '.', 'if', 'during', 'that', 'time', 'the', 'cattle-dealer', 'should', 'happen', 'to', 'call', 'and', 'want', 'to', 'buy', 'our', 'three']\n",
      "\n",
      "Processing file stories\\080.txt\n",
      "Data size (Characters) (Document 79) 588\n",
      "Sample string (Document 79) ['there', 'was', 'once', 'a', 'little', 'child', 'whose', 'mother', 'gave', 'her', 'every', 'afternoon', 'a', 'small', 'bowl', 'of', 'milk', 'and', 'bread', ',', 'and', 'the', 'child', 'seated', 'herself', 'in', 'the', 'yard', 'with', 'it', '.', 'but', 'when', 'she', 'began', 'to', 'eat', ',', 'a', 'paddock', 'came', 'creeping', 'out', 'of', 'a', 'crevice', 'in', 'the', 'wall', ',']\n",
      "\n",
      "Processing file stories\\081.txt\n",
      "Data size (Characters) (Document 80) 1670\n",
      "Sample string (Document 80) ['in', 'a', 'certain', 'mill', 'lived', 'an', 'old', 'miller', 'who', 'had', 'neither', 'wife', 'nor', 'child', ',', 'and', 'three', 'apprentices', 'served', 'under', 'him', '.', 'as', 'they', 'had', 'been', 'with', 'him', 'several', 'years', ',', 'he', 'one', 'day', 'said', 'to', 'them', ',', '``', 'i', 'am', 'old', ',', 'and', 'want', 'to', 'sit', 'behind', 'the', 'stove']\n",
      "\n",
      "Processing file stories\\082.txt\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Data size (Characters) (Document 81) 4887\n",
      "Sample string (Document 81) ['hill', 'and', 'vale', 'do', 'not', 'meet', ',', 'but', 'the', 'children', 'of', 'men', 'do', ',', 'good', 'and', 'bad', '.', 'in', 'this', 'way', 'a', 'shoemaker', 'and', 'a', 'tailor', 'once', 'met', 'on', 'their', 'travels', '.', 'the', 'tailor', 'was', 'a', 'handsome', 'little', 'fellow', 'who', 'was', 'always', 'merry', 'and', 'full', 'of', 'enjoyment', '.', 'he', 'saw']\n",
      "\n",
      "Processing file stories\\083.txt\n",
      "Data size (Characters) (Document 82) 2435\n",
      "Sample string (Document 82) ['hans', 'the', 'hedgehog', 'there', 'was', 'once', 'a', 'country', 'man', 'who', 'had', 'money', 'and', 'land', 'in', 'plenty', ',', 'but', 'however', 'rich', 'he', 'was', ',', 'his', 'happiness', 'was', 'still', 'lacking', 'in', 'one', 'respect', '-', 'he', 'had', 'no', 'children', '.', 'often', 'when', 'he', 'went', 'into', 'the', 'town', 'with', 'the', 'other', 'peasants', 'they', 'mocked']\n",
      "\n",
      "Processing file stories\\084.txt\n",
      "Data size (Characters) (Document 83) 298\n",
      "Sample string (Document 83) ['there', 'was', 'once', 'a', 'mother', 'who', 'had', 'a', 'little', 'boy', 'of', 'seven', 'years', 'old', ',', 'who', 'was', 'so', 'handsome', 'and', 'lovable', 'that', 'no', 'one', 'could', 'look', 'at', 'him', 'without', 'liking', 'him', ',', 'and', 'she', 'herself', 'worshipped', 'him', 'above', 'everything', 'in', 'the', 'world', '.', 'now', 'it', 'so', 'happened', 'that', 'he', 'suddenly']\n",
      "\n",
      "Processing file stories\\085.txt\n",
      "Data size (Characters) (Document 84) 2721\n",
      "Sample string (Document 84) ['there', 'was', 'once', 'a', 'young', 'fellow', 'who', 'had', 'learnt', 'the', 'trade', 'of', 'locksmith', ',', 'and', 'told', 'his', 'father', 'he', 'would', 'now', 'go', 'out', 'into', 'the', 'world', 'and', 'seek', 'his', 'fortune', '.', 'very', 'well', ',', 'said', 'the', 'father', ',', 'i', 'am', 'quite', 'content', 'with', 'that', ',', 'and', 'gave', 'him', 'some', 'money']\n",
      "\n",
      "Processing file stories\\086.txt\n",
      "Data size (Characters) (Document 85) 4011\n",
      "Sample string (Document 85) ['there', 'was', 'once', 'upon', 'a', 'time', 'a', 'king', 'who', 'had', 'a', 'little', 'boy', 'in', 'whose', 'stars', 'it', 'had', 'been', 'foretold', 'that', 'he', 'should', 'be', 'killed', 'by', 'a', 'stag', 'when', 'he', 'was', 'sixteen', 'years', 'of', 'age', ',', 'and', 'when', 'he', 'had', 'reached', 'that', 'age', 'the', 'huntsmen', 'once', 'went', 'hunting', 'with', 'him']\n",
      "\n",
      "Processing file stories\\087.txt\n",
      "Data size (Characters) (Document 86) 1443\n",
      "Sample string (Document 86) ['there', 'was', 'once', 'upon', 'a', 'time', 'a', 'princess', 'who', 'was', 'extremely', 'proud', '.', 'if', 'a', 'wooer', 'came', 'she', 'gave', 'him', 'some', 'riddle', 'to', 'guess', ',', 'and', 'if', 'he', 'could', 'not', 'guess', 'it', ',', 'he', 'was', 'sent', 'contemptuously', 'away', '.', 'she', 'let', 'it', 'be', 'made', 'known', 'also', 'that', 'whosoever', 'solved', 'her']\n",
      "\n",
      "Processing file stories\\088.txt\n",
      "Data size (Characters) (Document 87) 628\n",
      "Sample string (Document 87) ['a', 'tailor', \"'s\", 'apprentice', 'was', 'traveling', 'about', 'the', 'world', 'in', 'search', 'of', 'work', ',', 'and', 'at', 'one', 'time', 'he', 'could', 'find', 'none', ',', 'and', 'his', 'poverty', 'was', 'so', 'great', 'that', 'he', 'had', 'not', 'a', 'farthing', 'to', 'live', 'on', '.', 'presently', 'he', 'met', 'a', 'jew', 'on', 'the', 'road', ',', 'and', 'as']\n",
      "\n",
      "Processing file stories\\089.txt\n",
      "Data size (Characters) (Document 88) 2099\n",
      "Sample string (Document 88) ['there', 'was', 'once', 'on', 'a', 'time', 'a', 'soldier', 'who', 'for', 'many', 'years', 'had', 'served', 'the', 'king', 'faithfully', ',', 'but', 'when', 'the', 'war', 'came', 'to', 'an', 'end', 'could', 'serve', 'no', 'longer', 'because', 'of', 'the', 'many', 'wounds', 'which', 'he', 'had', 'received', '.', 'the', 'king', 'said', 'to', 'him', ',', '``', 'you', 'may', 'return']\n",
      "\n",
      "Processing file stories\\090.txt\n",
      "Data size (Characters) (Document 89) 160\n",
      "Sample string (Document 89) ['once', 'upon', 'a', 'time', 'there', 'was', 'a', 'child', 'who', 'was', 'willful', ',', 'and', 'would', 'not', 'do', 'what', 'her', 'mother', 'wished', '.', 'for', 'this', 'reason', 'god', 'had', 'no', 'pleasure', 'in', 'her', ',', 'and', 'let', 'her', 'become', 'ill', ',', 'and', 'no', 'doctor', 'could', 'do', 'her', 'any', 'good', ',', 'and', 'in', 'a', 'short']\n",
      "\n",
      "Processing file stories\\091.txt\n",
      "Data size (Characters) (Document 90) 2518\n",
      "Sample string (Document 90) ['there', 'was', 'once', 'a', 'king', \"'s\", 'son', ',', 'who', 'was', 'no', 'longer', 'content', 'to', 'stay', 'at', 'home', 'in', 'his', 'father', \"'s\", 'house', ',', 'and', 'as', 'he', 'had', 'no', 'fear', 'of', 'anything', ',', 'he', 'thought', ',', 'i', 'will', 'go', 'forth', 'into', 'the', 'wide', 'world', ',', 'there', 'the', 'time', 'will', 'not', 'seem']\n",
      "\n",
      "Processing file stories\\092.txt\n",
      "Data size (Characters) (Document 91) 3120\n",
      "Sample string (Document 91) ['there', 'was', 'once', 'a', 'young', 'huntsman', 'who', 'went', 'into', 'the', 'forest', 'to', 'lie', 'in', 'wait', '.', 'he', 'had', 'a', 'fresh', 'and', 'joyous', 'heart', ',', 'and', 'as', 'he', 'was', 'going', 'thither', ',', 'whistling', 'upon', 'a', 'leaf', ',', 'an', 'ugly', 'old', 'crone', 'came', 'up', ',', 'who', 'spoke', 'to', 'him', 'and', 'said', ',']\n",
      "\n",
      "Processing file stories\\093.txt\n",
      "Data size (Characters) (Document 92) 1077\n",
      "Sample string (Document 92) ['a', 'poor', 'servant-girl', 'was', 'once', 'traveling', 'with', 'the', 'family', 'with', 'which', 'she', 'was', 'in', 'service', ',', 'through', 'a', 'great', 'forest', ',', 'and', 'when', 'they', 'were', 'in', 'the', 'midst', 'of', 'it', ',', 'robbers', 'came', 'out', 'of', 'the', 'thicket', ',', 'and', 'murdered', 'all', 'they', 'found', '.', 'all', 'perished', 'together', 'except', 'the', 'girl']\n",
      "\n",
      "Processing file stories\\094.txt\n",
      "Data size (Characters) (Document 93) 778\n",
      "Sample string (Document 93) ['there', 'was', 'once', 'a', 'man', 'who', 'had', 'three', 'sons', ',', 'and', 'nothing', 'else', 'in', 'the', 'world', 'but', 'the', 'house', 'in', 'which', 'he', 'lived', '.', 'now', 'each', 'of', 'the', 'sons', 'wished', 'to', 'have', 'the', 'house', 'after', 'his', 'father', \"'s\", 'death', ',', 'but', 'the', 'father', 'loved', 'them', 'all', 'alike', ',', 'and', 'did']\n",
      "\n",
      "Processing file stories\\095.txt\n",
      "Data size (Characters) (Document 94) 1514\n",
      "Sample string (Document 94) ['there', 'was', 'a', 'great', 'war', ',', 'and', 'the', 'king', 'had', 'many', 'soldiers', ',', 'but', 'gave', 'them', 'small', 'pay', ',', 'so', 'small', 'that', 'they', 'could', 'not', 'live', 'upon', 'it', ',', 'so', 'three', 'of', 'them', 'agreed', 'among', 'themselves', 'to', 'desert', '.', 'one', 'of', 'them', 'said', 'to', 'the', 'others', ',', '``', 'if', 'we']\n",
      "\n",
      "Processing file stories\\096.txt\n",
      "Data size (Characters) (Document 95) 2369\n",
      "Sample string (Document 95) ['once', 'upon', 'a', 'time', 'lived', 'a', 'man', 'and', 'a', 'woman', 'who', 'so', 'long', 'as', 'they', 'were', 'rich', 'had', 'no', 'children', ',', 'but', 'when', 'they', 'were', 'poor', 'they', 'got', 'a', 'little', 'boy', '.', 'they', 'could', 'find', 'no', 'godfather', 'for', 'him', ',', 'so', 'the', 'man', 'said', 'he', 'would', 'just', 'go', 'to', 'another']\n",
      "\n",
      "Processing file stories\\097.txt\n",
      "Data size (Characters) (Document 96) 2613\n",
      "Sample string (Document 96) ['in', 'the', 'days', 'when', 'wishing', 'was', 'still', 'of', 'some', 'use', ',', 'a', 'king', \"'s\", 'son', 'was', 'bewitched', 'by', 'an', 'old', 'witch', ',', 'and', 'shut', 'up', 'in', 'an', 'iron', 'stove', 'in', 'a', 'forest', '.', 'there', 'he', 'passed', 'many', 'years', ',', 'and', 'no', 'one', 'could', 'rescue', 'him', '.', 'then', 'a', 'king', \"'s\"]\n",
      "\n",
      "Processing file stories\\098.txt\n",
      "Data size (Characters) (Document 97) 1980\n",
      "Sample string (Document 97) ['there', 'was', 'once', 'a', 'poor', 'man', 'who', 'had', 'four', 'sons', ',', 'and', 'when', 'they', 'were', 'grown', 'up', ',', 'he', 'said', 'to', 'them', ',', '``', 'my', 'dear', 'children', ',', 'you', 'must', 'now', 'go', 'out', 'into', 'the', 'world', ',', 'for', 'i', 'have', 'nothing', 'to', 'give', 'you', ',', 'so', 'set', 'out', ',', 'go']\n",
      "\n",
      "Processing file stories\\099.txt\n",
      "Data size (Characters) (Document 98) 3169\n",
      "Sample string (Document 98) ['there', 'was', 'once', 'a', 'woman', 'who', 'had', 'three', 'daughters', ',', 'the', 'eldest', 'of', 'whom', 'was', 'called', 'one-eye', ',', 'because', 'she', 'had', 'only', 'one', 'eye', 'in', 'the', 'middle', 'of', 'her', 'forehead', ',', 'and', 'the', 'second', ',', 'two-eyes', ',', 'because', 'she', 'had', 'two', 'eyes', 'like', 'other', 'folks', ',', 'and', 'the', 'youngest', ',']\n",
      "\n",
      "Processing file stories\\100.txt\n",
      "Data size (Characters) (Document 99) 492\n",
      "Sample string (Document 99) ['``', 'good-day', ',', 'father', 'hollenthe', '.', \"''\", '``', 'many', 'thanks', ',', 'pif-paf-poltrie', '.', \"''\", '``', 'may', 'i', 'be', 'allowed', 'to', 'have', 'your', 'daughter', '?', \"''\", '``', 'oh', ',', 'yes', ',', 'if', 'mother', 'malcho', 'milchcow', ',', 'brother', 'high-and-mighty', ',', 'sister', 'kasetraut', ',', 'and', 'fair', 'katrinelje', 'are', 'willing', ',', 'you', 'can', 'have']\n"
     ]
    }
   ],
   "source": [
    "def read_data(filename):\n",
    "  \n",
    "  with open(filename) as f:\n",
    "    data = tf.compat.as_str(f.read())\n",
    "    data = data.lower()\n",
    "    data = nltk.word_tokenize(data)\n",
    "    \n",
    "  return data\n",
    "\n",
    "documents = []\n",
    "global documents\n",
    "for i in range(num_files):    \n",
    "    print('\\nProcessing file %s'%os.path.join(dir_name,filenames[i]))\n",
    "    \n",
    "    # Unlike in the previous instances we break the text in to words\n",
    "    # this time\n",
    "    words = read_data(os.path.join(dir_name,filenames[i]))\n",
    "    \n",
    "    documents.append(words)\n",
    "    print('Data size (Characters) (Document %d) %d' %(i,len(words)))\n",
    "    print('Sample string (Document %d) %s'%(i,words[:50]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Building the Dictionaries (Bigrams)\n",
    "Builds the following. To understand each of these elements, let us also assume the text \"I like to go to school\"\n",
    "\n",
    "* `dictionary`: maps a string word to an ID (e.g. {I:0, like:1, to:2, go:3, school:4})\n",
    "* `reverse_dictionary`: maps an ID to a string word (e.g. {0:I, 1:like, 2:to, 3:go, 4:school}\n",
    "* `count`: List of list of (word, frequency) elements (e.g. [(I,1),(like,1),(to,2),(go,1),(school,1)]\n",
    "* `data` : Contain the string of text we read, where string words are replaced with word IDs (e.g. [0, 1, 2, 3, 2, 4])\n",
    "\n",
    "It also introduces an additional special token `UNK` to denote rare words to are too rare to make use of."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "202036 Words found.\n",
      "Most common words (+UNK) [(',', 17882), ('the', 11960), ('and', 9325), ('.', 7633), ('to', 4520)]\n",
      "Least common words (+UNK) [('admiring', 1), ('bearing', 1), ('scared', 1), ('treating', 1), ('gnome', 1), ('insolence', 1), ('disturber', 1), (\"carpenter's\", 1), ('brew', 1), ('hesitated', 1), ('summer-time', 1), ('exact', 1), ('window-frame', 1), ('reckoned', 1), ('hindered', 1)]\n",
      "Sample data [12, 0, 790, 25, 1237, 118, 831, 41, 1, 44]\n",
      "Sample data [573, 60, 7, 83, 120, 0, 7, 0, 22, 19]\n",
      "Vocabulary:  1322\n"
     ]
    }
   ],
   "source": [
    "def build_dataset(documents):\n",
    "    chars = []\n",
    "    # This is going to be a list of lists\n",
    "    # Where the outer list denote each document\n",
    "    # and the inner lists denote words in a given document\n",
    "    data_list = []\n",
    "  \n",
    "    for d in documents:\n",
    "        chars.extend(d)\n",
    "    print('%d Words found.'%len(chars))\n",
    "    count = []\n",
    "    # Get the word sorted by their frequency (Highest comes first)\n",
    "    count.extend(collections.Counter(chars).most_common())\n",
    "    \n",
    "    # Create an ID for each word by giving the current length of the dictionary\n",
    "    # And adding that item to the dictionary\n",
    "    # Start with 'UNK' that is assigned to too rare words\n",
    "    dictionary = dict({'UNK':0})\n",
    "    for char, c in count:\n",
    "        # Only add a bigram to dictionary if its frequency is more than 10\n",
    "        if c > 10:\n",
    "            dictionary[char] = len(dictionary)    \n",
    "    \n",
    "    unk_count = 0\n",
    "    # Traverse through all the text we have\n",
    "    # to replace each string word with the ID of the word\n",
    "    for d in documents:\n",
    "        data = list()\n",
    "        for char in d:\n",
    "            # If word is in the dictionary use the word ID,\n",
    "            # else use the ID of the special token \"UNK\"\n",
    "            if char in dictionary:\n",
    "                index = dictionary[char]        \n",
    "            else:\n",
    "                index = dictionary['UNK']\n",
    "                unk_count += 1\n",
    "            data.append(index)\n",
    "            \n",
    "        data_list.append(data)\n",
    "        \n",
    "    reverse_dictionary = dict(zip(dictionary.values(), dictionary.keys())) \n",
    "    return data_list, count, dictionary, reverse_dictionary\n",
    "\n",
    "global data_list, count, dictionary, reverse_dictionary,vocabulary_size\n",
    "\n",
    "# Print some statistics about data\n",
    "data_list, count, dictionary, reverse_dictionary = build_dataset(documents)\n",
    "print('Most common words (+UNK)', count[:5])\n",
    "print('Least common words (+UNK)', count[-15:])\n",
    "print('Sample data', data_list[0][:10])\n",
    "print('Sample data', data_list[1][:10])\n",
    "print('Vocabulary: ',len(dictionary))\n",
    "vocabulary_size = len(dictionary)\n",
    "del documents  # To reduce memory."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## CBOW: Learning Word Vectors\n",
    "\n",
    "In this section we learn word vectors using CBOW algorithm. This process can take a long time to run (~30 mins). Therefore, we have saved a version of final embeddings learnt by the algorithm. This will be loaded straight from the disk during text generation training. Therefore, you don't have to run this part. However, we have included this for the sake of completeness"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "with window_size = 1:\n",
      "    batch: [['in', 'times'], ['UNK', 'when'], ['times', 'wishing'], ['when', 'still'], ['wishing', 'helped'], ['still', 'one'], ['helped', ','], ['one', 'there']]\n",
      "    labels: ['UNK', 'times', 'when', 'wishing', 'still', 'helped', 'one', ',']\n",
      "\n",
      "with window_size = 2:\n",
      "    batch: [['a', 'king', 'daughters', 'were'], ['king', 'whose', 'were', 'all'], ['whose', 'daughters', 'all', 'beautiful'], ['daughters', 'were', 'beautiful', ','], ['were', 'all', ',', 'but'], ['all', 'beautiful', 'but', 'the'], ['beautiful', ',', 'the', 'youngest'], [',', 'but', 'youngest', 'was']]\n",
      "    labels: ['whose', 'daughters', 'were', 'all', 'beautiful', ',', 'but', 'the']\n",
      "Defining 6 embedding lookups representing each word in the context\n",
      "Stacked embedding size: [128, 128, 6]\n",
      "Reduced mean embedding size: [128, 128]\n",
      "WARNING:tensorflow:From c:\\users\\thushan\\documents\\python_virtualenvs\\tensorflow_venv\\lib\\site-packages\\tensorflow\\python\\ops\\nn_impl.py:1344: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See @{tf.nn.softmax_cross_entropy_with_logits_v2}.\n",
      "\n",
      "Initialized\n",
      "Average loss at step 1: 28208.264590\n",
      "Average loss at step 2: 2.362424\n",
      "Average loss at step 3: 2.194255\n",
      "Average loss at step 4: 46.048952\n",
      "Average loss at step 5: 2.404680\n",
      "Nearest to that: why, how, what, divided,\n",
      "Nearest to and: however, though, therefore, dark,\n",
      "Nearest to with: instantly, wherein, fetch, give,\n",
      "Nearest to had: has, knows, entered, knew,\n",
      "Nearest to will: can, may, shall, must,\n",
      "Nearest to one: longer, milk, sleeping, fear,\n",
      "Nearest to there: outside, whilst, bad, behold,\n",
      "Nearest to came: reached, went, comes, spoke,\n",
      "Nearest to said: replied, cried, answered, asked,\n",
      "Nearest to they: we, happily, she, iron,\n",
      "Nearest to but: however, therefore, fish-pond, though,\n",
      "Nearest to them: himself, him, themselves, creep,\n",
      "Nearest to my: your, his, its, our,\n",
      "Nearest to to: let, towards, comforted, told,\n",
      "Nearest to went: goes, rode, drove, ran,\n",
      "Nearest to ``: dear, to-day, allow, alas,\n",
      "Average loss at step 6: 6.750700\n",
      "Average loss at step 7: 4.194469\n",
      "Average loss at step 8: 3.253126\n",
      "Average loss at step 9: 1.641376\n",
      "Average loss at step 10: 3.060032\n",
      "Nearest to that: how, why, divided, where,\n",
      "Nearest to and: however, though, therefore, wherein,\n",
      "Nearest to with: instantly, tied, offered, wherein,\n",
      "Nearest to had: has, felt, have, wore,\n",
      "Nearest to will: may, can, shall, 'll,\n",
      "Nearest to one: couple, shone, everyone, dove,\n",
      "Nearest to there: outside, bad, yours, stupid,\n",
      "Nearest to came: reached, comes, ran, drew,\n",
      "Nearest to said: replied, answered, cried, spoke,\n",
      "Nearest to they: we, iron, happily, sisters,\n",
      "Nearest to but: therefore, 'ah, however, though,\n",
      "Nearest to them: him, himself, other, trees,\n",
      "Nearest to my: your, his, our, its,\n",
      "Nearest to to: towards, might, willingly, fulfilled,\n",
      "Nearest to went: goes, rode, drove, ran,\n",
      "Nearest to ``: allow, dear, alas, bad,\n"
     ]
    }
   ],
   "source": [
    "embedding_size = 128 # Dimension of the embedding vector.\n",
    "\n",
    "word2vec.define_data_and_hyperparameters(\n",
    "    num_files, data_list, reverse_dictionary, embedding_size, vocabulary_size)\n",
    "word2vec.print_some_batches()\n",
    "word2vec.define_word2vec_tensorflow()\n",
    "\n",
    "# We save the resulting embeddings as embeddings-tmp.npy \n",
    "# If you want to use this embedding for the following steps\n",
    "# please change the name to embeddings.npy and replace the existing\n",
    "word2vec.run_word2vec()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Generating Batches of Data\n",
    "The following object generates a batch of data which will be used to train the LSTM. More specifically the generator breaks a given sequence of words into `batch_size` segments. We also maintain a cursor for each segment. So whenever we create a batch of data, we sample one item from each segment and update the cursor of each segment. "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Unrolled index 0\n",
      "\tInputs:\n",
      "\tthat (14), \twhich (56), \t, (1), \tshone (808), \tclose (849), \n",
      "\tOutput:\n",
      "\tthe (2), \thas (114), \twas (8), \tin (12), \tby (60), \n",
      "\n",
      "Unrolled index 1\n",
      "\tInputs:\n",
      "\tthe (2), \thas (114), \twas (8), \tin (12), \tby (60), \n",
      "\tOutput:\n",
      "\tsun (403), \tseen (284), \tastonished (876), \ther (17), \tthe (2), \n",
      "\n",
      "Unrolled index 2\n",
      "\tInputs:\n",
      "\tsun (403), \tseen (284), \tastonished (876), \ther (17), \tthe (2), \n",
      "\tOutput:\n",
      "\titself (700), \tso (33), \twhenever (1319), \tface (361), \tking (34), \n",
      "\n",
      "Unrolled index 3\n",
      "\tInputs:\n",
      "\titself (700), \tso (33), \twhenever (1319), \tface (361), \tking (34), \n",
      "\tOutput:\n",
      "\t, (1), \tmuch (165), \tit (9), \t. (4), \t's (53), \n",
      "\n",
      "Unrolled index 4\n",
      "\tInputs:\n",
      "\t, (1), \tmuch (165), \tit (9), \t. (4), \tclose (849), \n",
      "\tOutput:\n",
      "\twhich (56), \t, (1), \tshone (808), \tclose (849), \tby (60), "
     ]
    }
   ],
   "source": [
    "class DataGeneratorSeq(object):\n",
    "    \n",
    "    def __init__(self,text,batch_size,num_unroll):\n",
    "        # Text where a bigram is denoted by its ID\n",
    "        self._text = text\n",
    "        # Number of bigrams in the text\n",
    "        self._text_size = len(self._text)\n",
    "        # Number of datapoints in a batch of data\n",
    "        self._batch_size = batch_size\n",
    "        # Num unroll is the number of steps we unroll the RNN in a single training step\n",
    "        # This relates to the truncated backpropagation we discuss in Chapter 6 text\n",
    "        self._num_unroll = num_unroll\n",
    "        # We break the text in to several segments and the batch of data is sampled by\n",
    "        # sampling a single item from a single segment\n",
    "        self._segments = self._text_size//self._batch_size\n",
    "        self._cursor = [offset * self._segments for offset in range(self._batch_size)]\n",
    "        \n",
    "    def next_batch(self):\n",
    "        '''\n",
    "        Generates a single batch of data\n",
    "        '''\n",
    "        # Train inputs (one-hot-encoded) and train outputs (one-hot-encoded)\n",
    "        batch_data = np.zeros((self._batch_size),dtype=np.float32)\n",
    "        batch_labels = np.zeros((self._batch_size, vocabulary_size),dtype=np.float32)\n",
    "        \n",
    "        # Fill in the batch datapoint by datapoint\n",
    "        for b in range(self._batch_size):\n",
    "            # If the cursor of a given segment exceeds the segment length\n",
    "            # we reset the cursor back to the beginning of that segment\n",
    "            if self._cursor[b]+1>=self._text_size:\n",
    "                self._cursor[b] = b * self._segments\n",
    "            \n",
    "            # Add the text at the cursor as the input\n",
    "            batch_data[b] = self._text[self._cursor[b]]\n",
    "            # Add the preceding bigram as the label to be predicted\n",
    "            batch_labels[b,self._text[self._cursor[b]+1]]= 1.0                       \n",
    "            # Update the cursor\n",
    "            self._cursor[b] = (self._cursor[b]+1)%self._text_size\n",
    "                    \n",
    "        return batch_data,batch_labels\n",
    "        \n",
    "    def unroll_batches(self):\n",
    "        '''\n",
    "        This produces a list of num_unroll batches\n",
    "        as required by a single step of training of the RNN\n",
    "        '''\n",
    "        unroll_data,unroll_labels = [],[]\n",
    "        for ui in range(self._num_unroll):\n",
    "            data, labels = self.next_batch()            \n",
    "            unroll_data.append(data)\n",
    "            unroll_labels.append(labels)\n",
    "        \n",
    "        return unroll_data, unroll_labels\n",
    "    \n",
    "    def reset_indices(self):\n",
    "        '''\n",
    "        Used to reset all the cursors if needed\n",
    "        '''\n",
    "        self._cursor = [offset * self._segments for offset in range(self._batch_size)]\n",
    "        \n",
    "# Running a tiny set to see if things are correct\n",
    "dg = DataGeneratorSeq(data_list[0][25:50],5,5)\n",
    "u_data, u_labels = dg.unroll_batches()\n",
    "\n",
    "# Iterate through each data batch in the unrolled set of batches\n",
    "for ui,(dat,lbl) in enumerate(zip(u_data,u_labels)):   \n",
    "    print('\\n\\nUnrolled index %d'%ui)\n",
    "    dat_ind = dat\n",
    "    lbl_ind = np.argmax(lbl,axis=1)\n",
    "    print('\\tInputs:')\n",
    "    for sing_dat in dat_ind:\n",
    "        print('\\t%s (%d)'%(reverse_dictionary[sing_dat],sing_dat),end=\", \")\n",
    "    print('\\n\\tOutput:')\n",
    "    for sing_lbl in lbl_ind:        \n",
    "        print('\\t%s (%d)'%(reverse_dictionary[sing_lbl],sing_lbl),end=\", \")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defining the LSTM\n",
    "\n",
    "This is a standard LSTM. The LSTM has 5 main components.\n",
    "* Cell state\n",
    "* Hidden state\n",
    "* Input gate\n",
    "* Forget gate\n",
    "* Output gate\n",
    "\n",
    "Each gate has three sets of weights (1 set for the current input, 1 set for the previous hidden state and 1 bias)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defining hyperparameters\n",
    "\n",
    "Here we define several hyperparameters and are very similar to the ones we defined in Chapter 6. However additionally we use dropout; a technique that helps to avoid overfitting."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Number of neurons in the hidden state variables\n",
    "num_nodes = 128\n",
    "\n",
    "# Number of data points in a batch we process\n",
    "batch_size = 64\n",
    "\n",
    "# Number of time steps we unroll for during optimization\n",
    "num_unrollings = 50\n",
    "\n",
    "dropout = 0.2 # We use dropout\n",
    "\n",
    "# Use this in the CSV filename when saving\n",
    "# when using dropout\n",
    "filename_extension = ''\n",
    "if dropout>0.0:\n",
    "    filename_extension = '_dropout'\n",
    "    \n",
    "filename_to_save = 'lstm_word2vec'+filename_extension + '.csv' # use to save perplexity values"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defining Inputs and Outputs\n",
    "\n",
    "In the code we define two different types of inputs. \n",
    "* Training inputs (The stories we downloaded) (batch_size > 1 with unrolling)\n",
    "* Validation inputs (An unseen validation dataset) (bach_size =1, no unrolling)\n",
    "* Test inputs (New story we are going to generate) (batch_size=1, no unrolling)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "tf.reset_default_graph()\n",
    "\n",
    "# Training Input data.\n",
    "train_inputs, train_labels = [],[]\n",
    "\n",
    "# Defining unrolled training inputs\n",
    "for ui in range(num_unrollings):\n",
    "    train_inputs.append(tf.placeholder(tf.int32, shape=[batch_size],name='train_inputs_%d'%ui))\n",
    "    train_labels.append(tf.placeholder(tf.float32, shape=[batch_size,vocabulary_size], name = 'train_labels_%d'%ui))\n",
    "\n",
    "# Validation data placeholders\n",
    "valid_inputs = tf.placeholder(tf.int32, shape=[1],name='valid_inputs')\n",
    "valid_labels = tf.placeholder(tf.float32, shape=[1,vocabulary_size], name = 'valid_labels')\n",
    "# Text generation: batch 1, no unrolling.\n",
    "test_input = tf.placeholder(tf.int32, shape=[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Loading Word Embeddings to TensorFlow\n",
    "We load the previously learned and stored embeddings to TensorFlow and define tensors to hold embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "## If you want to change the embedding matrix to something you newly generated,\n",
    "## Simply change embeddings.npy to embeddings-tmp.npy\n",
    "embed_mat = np.load('embeddings.npy')\n",
    "embed_init = tf.constant(embed_mat)\n",
    "embeddings = tf.Variable(embed_init,name='embeddings')\n",
    "embedding_size = embed_mat.shape[1]\n",
    "# Defining embedding lookup operations for all the unrolled\n",
    "# trianing inputs\n",
    "train_inputs_embeds = []\n",
    "for ui in range(num_unrollings):\n",
    "    train_inputs_embeds.append(tf.nn.embedding_lookup(embeddings,train_inputs[ui]))\n",
    "\n",
    "# Defining embedding lookup for operations for all the validation data\n",
    "valid_inputs_embeds = tf.nn.embedding_lookup(embeddings,valid_inputs)\n",
    "\n",
    "# Defining embedding lookup for operations for all the testing data\n",
    "test_input_embeds = tf.nn.embedding_lookup(embeddings, test_input)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defining Model Parameters and Cell Computation\n",
    "\n",
    "Now we define model parameters. Compared to RNNs, LSTMs have a large number of parameters. Each gate (input, forget, memory and output) has three different sets of parameters."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Input gate (i_t) - How much memory to write to cell state\n",
    "# Connects the current input to the input gate\n",
    "ix = tf.Variable(tf.truncated_normal([embedding_size, num_nodes], stddev=0.02))\n",
    "# Connects the previous hidden state to the input gate\n",
    "im = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], stddev=0.02))\n",
    "# Bias of the input gate\n",
    "ib = tf.Variable(tf.random_uniform([1, num_nodes],-0.02, 0.02))\n",
    "\n",
    "# Forget gate (f_t) - How much memory to discard from cell state\n",
    "# Connects the current input to the forget gate\n",
    "fx = tf.Variable(tf.truncated_normal([embedding_size, num_nodes], stddev=0.02))\n",
    "# Connects the previous hidden state to the forget gate\n",
    "fm = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], stddev=0.02))\n",
    "# Bias of the forget gate\n",
    "fb = tf.Variable(tf.random_uniform([1, num_nodes],-0.02, 0.02))\n",
    "\n",
    "# Candidate value (c~_t) - Used to compute the current cell state\n",
    "# Connects the current input to the candidate\n",
    "cx = tf.Variable(tf.truncated_normal([embedding_size, num_nodes], stddev=0.02))\n",
    "# Connects the previous hidden state to the candidate\n",
    "cm = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], stddev=0.02))\n",
    "# Bias of the candidate\n",
    "cb = tf.Variable(tf.random_uniform([1, num_nodes],-0.02,0.02))\n",
    "\n",
    "# Output gate - How much memory to output from the cell state\n",
    "# Connects the current input to the output gate\n",
    "ox = tf.Variable(tf.truncated_normal([embedding_size, num_nodes], stddev=0.02))\n",
    "# Connects the previous hidden state to the output gate\n",
    "om = tf.Variable(tf.truncated_normal([num_nodes, num_nodes], stddev=0.02))\n",
    "# Bias of the output gate\n",
    "ob = tf.Variable(tf.random_uniform([1, num_nodes],-0.02,0.02))\n",
    "\n",
    "\n",
    "# Softmax Classifier weights and biases.\n",
    "w = tf.Variable(tf.truncated_normal([num_nodes, vocabulary_size], stddev=0.02))\n",
    "b = tf.Variable(tf.random_uniform([vocabulary_size],-0.02,0.02))\n",
    "\n",
    "# Variables saving state across unrollings.\n",
    "# Hidden state\n",
    "saved_output = tf.Variable(tf.zeros([batch_size, num_nodes]), trainable=False)\n",
    "# Cell state\n",
    "saved_state = tf.Variable(tf.zeros([batch_size, num_nodes]), trainable=False)\n",
    "\n",
    "# Same variables for validation phase\n",
    "saved_valid_output = tf.Variable(tf.zeros([1, num_nodes]),trainable=False)\n",
    "saved_valid_state = tf.Variable(tf.zeros([1, num_nodes]),trainable=False)\n",
    "\n",
    "# Same variables for testing phase\n",
    "saved_test_output = tf.Variable(tf.zeros([1, num_nodes]),trainable=False)\n",
    "saved_test_state = tf.Variable(tf.zeros([1, num_nodes]),trainable=False)\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defining LSTM Computations\n",
    "Here first we define the LSTM cell computations as a consice function. Then we use this function to define training and test-time inference logic."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "algorithm = 'lstm_word2vec_beamsearch'\n",
    "filename_to_save = algorithm + filename_extension +'.csv'\n",
    "\n",
    "# Definition of the cell computation.\n",
    "def lstm_cell(i, o, state):\n",
    "    \"\"\"Create an LSTM cell\"\"\"\n",
    "    input_gate = tf.sigmoid(tf.matmul(i, ix) + tf.matmul(o, im) + ib)\n",
    "    forget_gate = tf.sigmoid(tf.matmul(i, fx) + tf.matmul(o, fm) + fb)\n",
    "    update = tf.matmul(i, cx) + tf.matmul(o, cm) + cb\n",
    "    state = forget_gate * state + input_gate * tf.tanh(update)\n",
    "    output_gate = tf.sigmoid(tf.matmul(i, ox) + tf.matmul(o, om) + ob)\n",
    "    return output_gate * tf.tanh(state), state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# =========================================================\n",
    "#Training related inference logic\n",
    "\n",
    "# Keeps the calculated state outputs in all the unrollings\n",
    "# Used to calculate loss\n",
    "outputs = list()\n",
    "\n",
    "# These two python variables are iteratively updated\n",
    "# at each step of unrolling\n",
    "output = saved_output\n",
    "state = saved_state\n",
    "\n",
    "# Compute the hidden state (output) and cell state (state)\n",
    "# recursively for all the steps in unrolling\n",
    "for i in train_inputs_embeds:\n",
    "    output, state = lstm_cell(i, output, state)\n",
    "    output = tf.nn.dropout(output,keep_prob=1.0-dropout)\n",
    "    # Append each computed output value\n",
    "    outputs.append(output)\n",
    "\n",
    "# calculate the score values\n",
    "logits = tf.matmul(tf.concat(axis=0, values=outputs), w) + b\n",
    "    \n",
    "# Compute predictions.\n",
    "train_prediction = tf.nn.softmax(logits)\n",
    "\n",
    "# Compute training perplexity\n",
    "train_perplexity_without_exp = tf.reduce_sum(tf.concat(train_labels,0)*-tf.log(tf.concat(train_prediction,0)+1e-10))/(num_unrollings*batch_size)\n",
    "\n",
    "# ========================================================================\n",
    "# Validation phase related inference logic\n",
    "\n",
    "# Compute the LSTM cell output for validation data\n",
    "valid_output, valid_state = lstm_cell(\n",
    "    valid_inputs_embeds, saved_valid_output, saved_valid_state)\n",
    "\n",
    "valid_logits = tf.nn.xw_plus_b(valid_output, w, b)\n",
    "\n",
    "# Make sure that the state variables are updated\n",
    "# before moving on to the next iteration of generation\n",
    "with tf.control_dependencies([saved_valid_output.assign(valid_output),\n",
    "                            saved_valid_state.assign(valid_state)]):\n",
    "    valid_prediction = tf.nn.softmax(valid_logits)\n",
    "\n",
    "# Compute validation perplexity\n",
    "valid_perplexity_without_exp = tf.reduce_sum(valid_labels*-tf.log(valid_prediction+1e-10))\n",
    "\n",
    "# ========================================================================\n",
    "# Testing phase related inference logic\n",
    "\n",
    "# Compute the LSTM cell output for testing data\n",
    "test_output, test_state = lstm_cell(\n",
    "test_input_embeds, saved_test_output, saved_test_state)\n",
    "\n",
    "# Make sure that the state variables are updated\n",
    "# before moving on to the next iteration of generation\n",
    "with tf.control_dependencies([saved_test_output.assign(test_output),\n",
    "                            saved_test_state.assign(test_state)]):\n",
    "    test_prediction = tf.nn.softmax(tf.nn.xw_plus_b(test_output, w, b))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Calculating LSTM Loss\n",
    "We calculate the training loss of the LSTM here. It's a typical cross entropy loss calculated over all the scores we obtained for training data (`loss`)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Before calcualting the training loss,\n",
    "# save the hidden state and the cell state to\n",
    "# their respective TensorFlow variables\n",
    "with tf.control_dependencies([saved_output.assign(output),\n",
    "                            saved_state.assign(state)]):\n",
    "\n",
    "    # Calculate the training loss by\n",
    "    # concatenating the results from all the unrolled time steps\n",
    "    loss = tf.reduce_mean(\n",
    "      tf.nn.softmax_cross_entropy_with_logits_v2(\n",
    "        logits=logits, labels=tf.concat(axis=0, values=train_labels)))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Defining Learning Rate and the Optimizer with Gradient Clipping\n",
    "Here we define the learning rate and the optimizer we're going to use. We will be using the Adam optimizer as it is one of the best optimizers out there. Furthermore we use gradient clipping to prevent any gradient explosions."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Used for decaying learning rate\n",
    "gstep = tf.Variable(0, trainable=False)\n",
    "\n",
    "# Running this operation will cause the value of gstep\n",
    "# to increase, while in turn reducing the learning rate\n",
    "inc_gstep = tf.assign(gstep, gstep+1)\n",
    "\n",
    "# Adam Optimizer. And gradient clipping.\n",
    "tf_learning_rate = tf.train.exponential_decay(0.001,gstep,decay_steps=1, decay_rate=0.5)\n",
    "\n",
    "optimizer = tf.train.AdamOptimizer(tf_learning_rate)\n",
    "gradients, v = zip(*optimizer.compute_gradients(loss))\n",
    "# Clipping gradients\n",
    "gradients, _ = tf.clip_by_global_norm(gradients, 5.0)\n",
    "\n",
    "optimizer = optimizer.apply_gradients(\n",
    "    zip(gradients, v))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Resetting Operations for Resetting Hidden States\n",
    "Sometimes the state variable needs to be reset (e.g. when starting predictions at a beginning of a new epoch)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Reset train state\n",
    "reset_train_state = tf.group(tf.assign(saved_state, tf.zeros([batch_size, num_nodes])),\n",
    "                          tf.assign(saved_output, tf.zeros([batch_size, num_nodes])))\n",
    "\n",
    "# Reset valid state\n",
    "reset_valid_state = tf.group(tf.assign(saved_valid_state, tf.zeros([1, num_nodes])),\n",
    "                          tf.assign(saved_valid_output, tf.zeros([1, num_nodes])))\n",
    "\n",
    "# Reset test state\n",
    "reset_test_state = tf.group(\n",
    "    saved_test_output.assign(tf.random_normal([1, num_nodes],stddev=0.05)),\n",
    "    saved_test_state.assign(tf.random_normal([1, num_nodes],stddev=0.05)))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## LSTM with Beam-Search\n",
    "\n",
    "Here we alter the previously defined prediction related TensorFlow operations to employ beam-search. Beam search is a way of predicting several time steps ahead. Concretely instead of predicting the best prediction we have at a given time step, we get predictions for several time steps and get the sequence of highest joint probability."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "beam_length = 20\n",
    "beam_neighbors = 5\n",
    "\n",
    "# We redefine the sample generation with beam search\n",
    "sample_beam_inputs = [tf.placeholder(tf.int32, shape=[1]) for _ in range(beam_neighbors)]\n",
    "sample_input = tf.placeholder(tf.int32, shape=[1])\n",
    "\n",
    "# Embedding lookups for each beam\n",
    "sample_beam_inputs_embeds = [tf.nn.embedding_lookup(embeddings, inp) for inp in sample_beam_inputs]\n",
    "sample_input_embeds = tf.nn.embedding_lookup(embeddings, sample_input)\n",
    "\n",
    "best_beam_index = tf.placeholder(shape=None, dtype=tf.int32)\n",
    "best_neighbor_beam_indices = tf.placeholder(shape=[beam_neighbors], dtype=tf.int32)\n",
    "\n",
    "# Maintains output of each beam\n",
    "saved_sample_beam_output = [tf.Variable(tf.zeros([1, num_nodes])) for _ in range(beam_neighbors)]\n",
    "# Maintains the state of each beam\n",
    "saved_sample_beam_state = [tf.Variable(tf.zeros([1, num_nodes])) for _ in range(beam_neighbors)]\n",
    "\n",
    "# Resetting the sample beam states (should be done at the beginning of each text snippet generation)\n",
    "reset_sample_beam_state = tf.group(\n",
    "    *[saved_sample_beam_output[vi].assign(tf.zeros([1, num_nodes])) for vi in range(beam_neighbors)],\n",
    "    *[saved_sample_beam_state[vi].assign(tf.zeros([1, num_nodes])) for vi in range(beam_neighbors)]\n",
    ")\n",
    "\n",
    "# We stack them to perform gather operation below\n",
    "stacked_beam_outputs = tf.stack(saved_sample_beam_output)\n",
    "stacked_beam_states = tf.stack(saved_sample_beam_state)\n",
    "\n",
    "# The beam states for each beam (there are beam_neighbor-many beams) needs to be updated at every depth of tree\n",
    "# Consider an example where you have 3 classes where we get the best two neighbors (marked with star)\n",
    "#     a`      b*       c  \n",
    "#   / | \\   / | \\    / | \\\n",
    "#  a  b c  a* b` c  a  b  c\n",
    "# Since both the candidates from level 2 comes from the parent b\n",
    "# We need to update both states/outputs from saved_sample_beam_state/output to have index 1 (corresponding to parent b)\n",
    "update_sample_beam_state = tf.group(\n",
    "    *[saved_sample_beam_output[vi].assign(tf.gather_nd(stacked_beam_outputs,[best_neighbor_beam_indices[vi]])) for vi in range(beam_neighbors)],\n",
    "    *[saved_sample_beam_state[vi].assign(tf.gather_nd(stacked_beam_states,[best_neighbor_beam_indices[vi]])) for vi in range(beam_neighbors)]\n",
    ")\n",
    "\n",
    "# We calculate lstm_cell state and output for each beam\n",
    "sample_beam_outputs, sample_beam_states = [],[] \n",
    "for vi in range(beam_neighbors):\n",
    "    tmp_output, tmp_state = lstm_cell(\n",
    "        sample_beam_inputs_embeds[vi], saved_sample_beam_output[vi], saved_sample_beam_state[vi]\n",
    "    )\n",
    "    sample_beam_outputs.append(tmp_output)\n",
    "    sample_beam_states.append(tmp_state)\n",
    "\n",
    "# For a given set of beams, outputs a list of prediction vectors of size beam_neighbors\n",
    "# each beam having the predictions for full vocabulary\n",
    "sample_beam_predictions = []\n",
    "for vi in range(beam_neighbors):\n",
    "    with tf.control_dependencies([saved_sample_beam_output[vi].assign(sample_beam_outputs[vi]),\n",
    "                                saved_sample_beam_state[vi].assign(sample_beam_states[vi])]):\n",
    "        sample_beam_predictions.append(tf.nn.softmax(tf.nn.xw_plus_b(sample_beam_outputs[vi], w, b)))\n",
    "        "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Running the LSTM with Beam Search to Generate Text\n",
    "\n",
    "Here we train the LSTM on the available data and generate text using the trained LSTM for several steps. From each document we extract text for `steps_per_document` steps to train the LSTM on. We also report the train perplexity at the end of each step. Finally we test the LSTM by asking it to generate some new text with beam search starting from a randomly picked bigram."
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Learning rate Decay Logic\n",
    "\n",
    "Here we define the logic to decrease learning rate whenever the validation perplexity does not decrease"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Learning rate decay related\n",
    "# If valid perpelxity does not decrease\n",
    "# continuously for this many epochs\n",
    "# decrease the learning rate\n",
    "decay_threshold = 5\n",
    "# Keep counting perplexity increases\n",
    "decay_count = 0\n",
    "\n",
    "min_perplexity = 1e10\n",
    "\n",
    "# Learning rate decay logic\n",
    "def decay_learning_rate(session, v_perplexity):\n",
    "  global decay_threshold, decay_count, min_perplexity  \n",
    "  # Decay learning rate\n",
    "  if v_perplexity < min_perplexity:\n",
    "    decay_count = 0\n",
    "    min_perplexity= v_perplexity\n",
    "  else:\n",
    "    decay_count += 1\n",
    "\n",
    "  if decay_count >= decay_threshold:\n",
    "    print('\\t Reducing learning rate')\n",
    "    decay_count = 0\n",
    "    session.run(inc_gstep)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Defining the Beam Prediction Logic\n",
    "Here we define function that takes in the session as an argument and output a beam of predictions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_word = None\n",
    "\n",
    "def get_beam_prediction(session):\n",
    "    '''\n",
    "    Outputs a single beam of predictions of a specified length\n",
    "    '''\n",
    "    \n",
    "    # Generating words within a segment with Beam Search\n",
    "    # To make some calculations clearer, we use the example as follows\n",
    "    # We have three classes with beam_neighbors=2 (best candidate denoted by *, second best candidate denoted by `)\n",
    "    # For simplicity we assume best candidate always have probability of 0.5 in output prediction\n",
    "    # second best has 0.2 output prediction\n",
    "    #           a`                   b*                   c                <--- root level\n",
    "    #    /     |     \\         /     |     \\        /     |     \\   \n",
    "    #   a      b      c       a*     b`     c      a      b      c         <--- depth 1\n",
    "    # / | \\  / | \\  / | \\   / | \\  / | \\  / | \\  / | \\  / | \\  / | \\\n",
    "    # a b c  a b c  a b c   a*b c  a`b c  a b c  a b c  a b c  a b c       <--- depth 2\n",
    "    # So the best beams at depth 2 would be\n",
    "    # b-a-a and b-b-a\n",
    "    \n",
    "    global test_word\n",
    "    global sample_beam_predictions, update_sample_beam_state\n",
    "    \n",
    "    # Calculate the candidates at the root level\n",
    "    feed_dict = {}\n",
    "    for b_n_i in range(beam_neighbors):\n",
    "        feed_dict.update({sample_beam_inputs[b_n_i]: [test_word]})\n",
    "\n",
    "    # We calculate sample predictions for all neighbors with the same starting word/character\n",
    "    # This is important to update the state for all instances of beam search\n",
    "    sample_preds_root = session.run(sample_beam_predictions, feed_dict = feed_dict)  \n",
    "    sample_preds_root = sample_preds_root[0]\n",
    "\n",
    "    # indices of top-k candidates\n",
    "    # b and a in our example (root level)\n",
    "    this_level_candidates =  (np.argsort(sample_preds_root,axis=1).ravel()[::-1])[:beam_neighbors].tolist() \n",
    "    \n",
    "    # probabilities of top-k candidates\n",
    "    # 0.5 and 0.2\n",
    "    this_level_probs = sample_preds_root[0,this_level_candidates] #probabilities of top-k candidates\n",
    "\n",
    "    # Update test sequence produced by each beam from the root level calculation\n",
    "    # Test sequence looks like for our example (at root)\n",
    "    # [b,a]\n",
    "    test_sequences = ['' for _ in range(beam_neighbors)]\n",
    "    for b_n_i in range(beam_neighbors):\n",
    "        test_sequences[b_n_i] += reverse_dictionary[this_level_candidates[b_n_i]] + ' '\n",
    "\n",
    "    # Make the calculations for the rest of the depth of the beam search tree\n",
    "    for b_i in range(beam_length-1):\n",
    "        \n",
    "        test_words = [] # candidate words for each beam\n",
    "        pred_words = [] # Predicted words of each beam\n",
    "        \n",
    "        # computing feed_dict for the beam search (except root)\n",
    "        # feed dict should contain the best words/chars/bigrams found by the previous level of search\n",
    "\n",
    "        # For level 1 in our example this would be\n",
    "        # sample_beam_inputs[0]: b, sample_beam_inputs[1]:a\n",
    "        feed_dict = {}\n",
    "        for p_idx, pred_i in enumerate(this_level_candidates):                    \n",
    "            # Updating the feed_dict for getting next predictions\n",
    "            test_words.append(this_level_candidates[p_idx])\n",
    "\n",
    "            feed_dict.update({sample_beam_inputs[p_idx]:[test_words[p_idx]]})\n",
    "\n",
    "        # Calculating predictions for all neighbors in beams\n",
    "        # This is a list of vectors where each vector is the prediction vector for a certain beam\n",
    "        # For level 1 in our example, the prediction values for \n",
    "        #      b             a  (previous beam search results)\n",
    "        # [a,  b,  c],  [a,  b,  c] (current level predictions) would be\n",
    "        # [0.1,0.1,0.1],[0.5,0.2,0]\n",
    "        sample_preds_all_neighbors = session.run(sample_beam_predictions, feed_dict=feed_dict)\n",
    "\n",
    "        # Create a single vector with \n",
    "        # Making our example [0.1,0.1,0.1,0.5,0.2,0] \n",
    "        sample_preds_all_neighbors_concat = np.concatenate(sample_preds_all_neighbors,axis=1)\n",
    "        \n",
    "        # normalize this_level_candidates to fall between [0,vocabulary_size]\n",
    "        # In this example this would be [0,1]\n",
    "        this_level_candidates = np.argsort(sample_preds_all_neighbors_concat.ravel())[::-1][:beam_neighbors]\n",
    "        \n",
    "        # In the example this would be [1,1]\n",
    "        parent_beam_indices = this_level_candidates//vocabulary_size\n",
    "\n",
    "        # normalize this_level_candidates to fall between [0,vocabulary_size]\n",
    "        this_level_candidates = (this_level_candidates%vocabulary_size).tolist()\n",
    "\n",
    "        # Here we update the final state of each beam to be\n",
    "        # the state that was at the index 1. Because for both the candidates at this level the parent is \n",
    "        # at index 1 (that is b from root level)\n",
    "        session.run(update_sample_beam_state, feed_dict={best_neighbor_beam_indices: parent_beam_indices})\n",
    "\n",
    "        # Here we update the joint probabilities of each beam and add the newly found candidates to the sequence\n",
    "        tmp_this_level_probs = np.asarray(this_level_probs) # This is currently [0.5,0.2]\n",
    "        tmp_test_sequences = list(test_sequences) # This is currently [b,a]\n",
    "\n",
    "        for b_n_i in range(beam_neighbors):\n",
    "            # We make the b_n_i element of this_level_probs to be the probability of parents\n",
    "            # In the example the parent indices are [1,1]\n",
    "            # So this_level_probs become [0.5,0.5]\n",
    "            this_level_probs[b_n_i] = tmp_this_level_probs[parent_beam_indices[b_n_i]]\n",
    "            \n",
    "            # Next we multipyle these by the probabilities of the best candidates from current level \n",
    "            # [0.5*0.5, 0.5*0.2] = [0.25,0.1]\n",
    "            this_level_probs[b_n_i] *= sample_preds_all_neighbors[parent_beam_indices[b_n_i]][0,this_level_candidates[b_n_i]]\n",
    "\n",
    "            # Make the b_n_i element of test_sequences to be the correct parent of the current best candidates\n",
    "            # In the example this becomes [b, b]\n",
    "            test_sequences[b_n_i] = tmp_test_sequences[parent_beam_indices[b_n_i]]\n",
    "            \n",
    "            # Now we append the current best candidates\n",
    "            # In this example this becomes [ba,bb]\n",
    "            test_sequences[b_n_i] += reverse_dictionary[this_level_candidates[b_n_i]] + ' '\n",
    "\n",
    "            # Create one-hot-encoded representation for each candidate\n",
    "            pred_words.append(this_level_candidates[b_n_i])\n",
    "\n",
    "\n",
    "    # Calculate best beam id based on the highest beam probability\n",
    "    best_beam_id = parent_beam_indices[np.asscalar(np.argmax(this_level_probs))]\n",
    "\n",
    "    # Update state and output variables for test prediction\n",
    "    session.run(update_sample_beam_state,feed_dict={best_neighbor_beam_indices:[best_beam_id for _ in range(beam_neighbors)]})\n",
    "    \n",
    "    # Make the last word/character/bigram from the best beam\n",
    "    test_word = pred_words[best_beam_id]\n",
    "    \n",
    "    return test_sequences[best_beam_id]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Running Training, Validation and Generation\n",
    "\n",
    "We traing the LSTM on existing training data, check the validaiton perplexity on an unseen chunk of text and generate a fresh segment of text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Initialized\n",
      "Processing step: 0, Using learning rate: 0.00100\n",
      "(16).(14).(76).(97).(11).(64).(81).(80).(68).(99).\n",
      "Average loss at step 1: 3.941540\n",
      "\tPerplexity at step 1: 51.497868\n",
      "\n",
      "Valid Perplexity: 973.43\n",
      "\n",
      "Generated Text after epoch 0 ... \n",
      "======================== New text Segment ==========================\n",
      " in me , `` a UNK ? '' `` something better . '' `` something better . '' `` something better  , and a UNK ? '' `` something better . '' `` something better . '' `` something better .  are a UNK ? '' `` something better . '' `` something better . '' `` something better . ''  `` something better . '' `` something better . '' `` something better . '' `` something better . ''  `` something better . '' `` something better . '' `` something better . '' `` something better . ''  `` something better . '' `` something better . '' `` something better . '' `` something better . ''  `` something better . '' `` something better . '' `` something better . '' `` something better . ''  `` something better . '' `` something better . '' `` something better . '' `` something better . ''  `` something better . '' `` something better . '' `` something better . '' `` something better . ''  `` something better . '' `` something better . '' `` something better . '' `` something better . ''  `` something better . '' `` something better . '' `` something better . '' `` something better . ''  `` something better . '' `` something better . '' `` something better . '' `` something better . ''   \n",
      "====================================================================\n",
      "======================== New text Segment ==========================\n",
      " so '' then i am , and a UNK ? '' `` something better . '' `` something better . ''  `` something better . '' `` something better . '' `` something better . '' `` something better . ''  `` something better . '' `` something better . '' `` something better . '' `` something better . ''  `` something better . '' `` something better . '' `` something better . '' `` something better . ''  `` something better . '' `` something better . '' `` something better . '' `` something better . ''  `` something better . '' `` something better . '' `` something better . '' `` something better . ''  `` something better . '' `` something better . '' `` something better . '' `` something better . ''  `` something better . '' `` something better . '' `` something better . '' `` something better . ''  `` something better . '' `` something better . '' `` something better . '' `` something better . ''  `` something better . '' `` something better . '' `` something better . '' `` something better . ''  `` something better . '' `` something better . '' `` something better . '' `` something better . ''  `` something better . '' `` something better . '' `` something better . '' `` something better . ''   \n",
      "====================================================================\n",
      "\n",
      "Processing step: 1, Using learning rate: 0.00100\n",
      "(96).(9).(38).(20).(30).(15)."
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "\u001b[1;32m<ipython-input-19-b3737d5cdf61>\u001b[0m in \u001b[0;36m<module>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     55\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     56\u001b[0m             _, l, step_perplexity = session.run([optimizer, loss, train_perplexity_without_exp], \n\u001b[1;32m---> 57\u001b[1;33m                                                        feed_dict=feed_dict)\n\u001b[0m\u001b[0;32m     58\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m     59\u001b[0m             \u001b[0mdoc_perplexity\u001b[0m \u001b[1;33m+=\u001b[0m \u001b[0mstep_perplexity\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\thushan\\documents\\python_virtualenvs\\tensorflow_venv\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36mrun\u001b[1;34m(self, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m    898\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    899\u001b[0m       result = self._run(None, fetches, feed_dict, options_ptr,\n\u001b[1;32m--> 900\u001b[1;33m                          run_metadata_ptr)\n\u001b[0m\u001b[0;32m    901\u001b[0m       \u001b[1;32mif\u001b[0m \u001b[0mrun_metadata\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m    902\u001b[0m         \u001b[0mproto_data\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mtf_session\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mTF_GetBuffer\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mrun_metadata_ptr\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\thushan\\documents\\python_virtualenvs\\tensorflow_venv\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run\u001b[1;34m(self, handle, fetches, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1133\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mfinal_fetches\u001b[0m \u001b[1;32mor\u001b[0m \u001b[0mfinal_targets\u001b[0m \u001b[1;32mor\u001b[0m \u001b[1;33m(\u001b[0m\u001b[0mhandle\u001b[0m \u001b[1;32mand\u001b[0m \u001b[0mfeed_dict_tensor\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1134\u001b[0m       results = self._do_run(handle, final_targets, final_fetches,\n\u001b[1;32m-> 1135\u001b[1;33m                              feed_dict_tensor, options, run_metadata)\n\u001b[0m\u001b[0;32m   1136\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1137\u001b[0m       \u001b[0mresults\u001b[0m \u001b[1;33m=\u001b[0m \u001b[1;33m[\u001b[0m\u001b[1;33m]\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\thushan\\documents\\python_virtualenvs\\tensorflow_venv\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_run\u001b[1;34m(self, handle, target_list, fetch_list, feed_dict, options, run_metadata)\u001b[0m\n\u001b[0;32m   1314\u001b[0m     \u001b[1;32mif\u001b[0m \u001b[0mhandle\u001b[0m \u001b[1;32mis\u001b[0m \u001b[1;32mNone\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1315\u001b[0m       return self._do_call(_run_fn, feeds, fetches, targets, options,\n\u001b[1;32m-> 1316\u001b[1;33m                            run_metadata)\n\u001b[0m\u001b[0;32m   1317\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1318\u001b[0m       \u001b[1;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_do_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0m_prun_fn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeeds\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetches\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\thushan\\documents\\python_virtualenvs\\tensorflow_venv\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_do_call\u001b[1;34m(self, fn, *args)\u001b[0m\n\u001b[0;32m   1320\u001b[0m   \u001b[1;32mdef\u001b[0m \u001b[0m_do_call\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mself\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m,\u001b[0m \u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1321\u001b[0m     \u001b[1;32mtry\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1322\u001b[1;33m       \u001b[1;32mreturn\u001b[0m \u001b[0mfn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m*\u001b[0m\u001b[0margs\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0m\u001b[0;32m   1323\u001b[0m     \u001b[1;32mexcept\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mOpError\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0me\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1324\u001b[0m       \u001b[0mmessage\u001b[0m \u001b[1;33m=\u001b[0m \u001b[0mcompat\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mas_text\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0me\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mmessage\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\thushan\\documents\\python_virtualenvs\\tensorflow_venv\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_run_fn\u001b[1;34m(feed_dict, fetch_list, target_list, options, run_metadata)\u001b[0m\n\u001b[0;32m   1305\u001b[0m       \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_extend_graph\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1306\u001b[0m       return self._call_tf_sessionrun(\n\u001b[1;32m-> 1307\u001b[1;33m           options, feed_dict, fetch_list, target_list, run_metadata)\n\u001b[0m\u001b[0;32m   1308\u001b[0m \u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1309\u001b[0m     \u001b[1;32mdef\u001b[0m \u001b[0m_prun_fn\u001b[0m\u001b[1;33m(\u001b[0m\u001b[0mhandle\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m)\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;32mc:\\users\\thushan\\documents\\python_virtualenvs\\tensorflow_venv\\lib\\site-packages\\tensorflow\\python\\client\\session.py\u001b[0m in \u001b[0;36m_call_tf_sessionrun\u001b[1;34m(self, options, feed_dict, fetch_list, target_list, run_metadata)\u001b[0m\n\u001b[0;32m   1407\u001b[0m       return tf_session.TF_SessionRun_wrapper(\n\u001b[0;32m   1408\u001b[0m           \u001b[0mself\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0m_session\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0moptions\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfeed_dict\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mfetch_list\u001b[0m\u001b[1;33m,\u001b[0m \u001b[0mtarget_list\u001b[0m\u001b[1;33m,\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[1;32m-> 1409\u001b[1;33m           run_metadata)\n\u001b[0m\u001b[0;32m   1410\u001b[0m     \u001b[1;32melse\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n\u001b[0;32m   1411\u001b[0m       \u001b[1;32mwith\u001b[0m \u001b[0merrors\u001b[0m\u001b[1;33m.\u001b[0m\u001b[0mraise_exception_on_not_ok_status\u001b[0m\u001b[1;33m(\u001b[0m\u001b[1;33m)\u001b[0m \u001b[1;32mas\u001b[0m \u001b[0mstatus\u001b[0m\u001b[1;33m:\u001b[0m\u001b[1;33m\u001b[0m\u001b[0m\n",
      "\u001b[1;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "num_steps = 26\n",
    "steps_per_document = 100\n",
    "docs_per_step = 10\n",
    "valid_summary = 1\n",
    "train_doc_count = 100\n",
    "\n",
    "beam_nodes = []\n",
    "\n",
    "word2vec_train_perplexity_ot = []\n",
    "word2vec_valid_perplexity_ot = []\n",
    "\n",
    "session = tf.InteractiveSession()\n",
    "\n",
    "tf.global_variables_initializer().run()\n",
    "\n",
    "print('Initialized')\n",
    "average_loss = 0\n",
    "\n",
    "# We use the first 10 documents that has \n",
    "# more than (num_steps+1)*steps_per_document bigrams for creating the validation dataset\n",
    "\n",
    "# Identify the first 10 documents following the above condition\n",
    "long_doc_ids = []\n",
    "for di in range(num_files):\n",
    "  if len(data_list[di])>(num_steps+1)*steps_per_document:\n",
    "    long_doc_ids.append(di)\n",
    "  if len(long_doc_ids)==10:\n",
    "    break\n",
    "    \n",
    "# Generating data\n",
    "data_gens = []\n",
    "valid_gens = []\n",
    "for fi in range(num_files):\n",
    "  # Get all the bigrams if the document id is not in the validation document ids\n",
    "  if fi not in long_doc_ids:\n",
    "    data_gens.append(DataGeneratorSeq(data_list[fi],batch_size,num_unrollings))\n",
    "  # if the document is in the validation doc ids, only get up to the \n",
    "  # last steps_per_document bigrams and use the last steps_per_document bigrams as validation data\n",
    "  else:\n",
    "    data_gens.append(DataGeneratorSeq(data_list[fi][:-steps_per_document],batch_size,num_unrollings))\n",
    "    # Defining the validation data generator\n",
    "    valid_gens.append(DataGeneratorSeq(data_list[fi][-steps_per_document:],1,1))\n",
    "\n",
    "feed_dict = {}\n",
    "for step in range(num_steps):\n",
    "    print('Processing step: %d, Using learning rate: %.5f'%(step, session.run(tf_learning_rate)))\n",
    "    for di in np.random.permutation(train_doc_count)[:docs_per_step]:            \n",
    "        doc_perplexity = 0\n",
    "        for doc_step_id in range(steps_per_document):\n",
    "            \n",
    "            u_data, u_labels = data_gens[di].unroll_batches()\n",
    "            for ui,(dat,lbl) in enumerate(zip(u_data,u_labels)):            \n",
    "                feed_dict[train_inputs[ui]] = dat.reshape(-1).astype(np.int32)\n",
    "                feed_dict[train_labels[ui]] = lbl                \n",
    "            \n",
    "            _, l, step_perplexity = session.run([optimizer, loss, train_perplexity_without_exp], \n",
    "                                                       feed_dict=feed_dict)\n",
    "            \n",
    "            doc_perplexity += step_perplexity\n",
    "            \n",
    "            average_loss += step_perplexity\n",
    "            \n",
    "        \n",
    "        session.run(reset_train_state) # resetting hidden state for each document\n",
    "        \n",
    "        # Show the printing progress <train_doc_id_1>.<train_doc_id_2>. ...\n",
    "        print('(%d).'%di,end='') \n",
    "        \n",
    "    print('')\n",
    "    \n",
    "    if (step+1) % valid_summary == 0:\n",
    "      \n",
    "      average_loss = average_loss / (steps_per_document*docs_per_step*valid_summary)\n",
    "      \n",
    "      print('Average loss at step %d: %f' % (step+1, average_loss))\n",
    "      print('\\tPerplexity at step %d: %f' %(step+1, np.exp(average_loss)))\n",
    "      word2vec_train_perplexity_ot.append(np.exp(average_loss))\n",
    "      average_loss = 0 # reset loss\n",
    "      \n",
    "      valid_loss = 0 # reset loss\n",
    "        \n",
    "      # calculate valid perplexity\n",
    "      for v_doc_id in range(10):\n",
    "          # Remember we process things as bigrams\n",
    "          # So need to divide by 2\n",
    "          for v_step in range(steps_per_document//2):\n",
    "            uvalid_data,uvalid_labels = valid_gens[v_doc_id].unroll_batches()        \n",
    "\n",
    "            # Run validation phase related TensorFlow operations       \n",
    "            v_perp = session.run(\n",
    "                valid_perplexity_without_exp,\n",
    "                feed_dict = {valid_inputs:uvalid_data[0],valid_labels: uvalid_labels[0]}\n",
    "            )\n",
    "\n",
    "            valid_loss += v_perp\n",
    "            \n",
    "          session.run(reset_valid_state)\n",
    "      \n",
    "          # Reset validation data generator cursor\n",
    "          valid_gens[v_doc_id].reset_indices()      \n",
    "    \n",
    "      print()\n",
    "      v_perplexity = np.exp(valid_loss/(steps_per_document*10.0//2))\n",
    "      print(\"Valid Perplexity: %.2f\\n\"%v_perplexity)\n",
    "      word2vec_valid_perplexity_ot.append(v_perplexity)\n",
    "          \n",
    "      decay_learning_rate(session, v_perplexity)\n",
    "    \n",
    "      # Generating new text ...\n",
    "      # We will be generating one segment having 500 bigrams\n",
    "      # Feel free to generate several segments by changing\n",
    "      # the value of segments_to_generate\n",
    "    \n",
    "      print('Generated Text after epoch %d ... '%step)  \n",
    "      segments_to_generate = 2\n",
    "      chars_in_segment = 250//beam_length\n",
    "    \n",
    "      for _ in range(segments_to_generate):\n",
    "        print('======================== New text Segment ==========================')\n",
    "        # first word randomly generated\n",
    "        rand_doc = data_list[np.random.randint(0,num_files)]\n",
    "        test_word = rand_doc[np.random.randint(len(rand_doc))]\n",
    "        print(\"\",reverse_dictionary[test_word],end=' ')\n",
    "        \n",
    "        # Generating words within a segment with Beam Search\n",
    "        for _ in range(chars_in_segment):\n",
    "            \n",
    "            test_sequence = get_beam_prediction(session)\n",
    "            print(test_sequence,end=' ')\n",
    "            \n",
    "        print(\" \")\n",
    "        session.run(reset_sample_beam_state)\n",
    "        \n",
    "        print('====================================================================')\n",
    "      print(\"\")\n",
    "\n",
    "session.close()\n",
    "\n",
    "with open(filename_to_save, 'wt') as f:\n",
    "    writer = csv.writer(f, delimiter=',')\n",
    "    writer.writerow(word2vec_train_perplexity_ot)\n",
    "    writer.writerow(word2vec_valid_perplexity_ot)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
